After upgrading from 4.2 to 4.3.0 yesterday, I have also noticed that the rake task katello:delete_orphaned_content throws an exception. Both tasks, for the main server and for my content proxy.
Started like from cron:
[root@foreman ~]# foreman-rake katello:delete_orphaned_content RAILS_ENV=production
Orphaned content deletion started in background.
Orphaned content deletion started in background.
Katello::Errors::Pulp3Error: ("Cannot delete some instances of model 'Repository' because they are referenced through protected foreign keys: 'Variant.repository', 'RpmRepository.repository_ptr'.", {<Variant: AppStream>})
Output:
{"pulp_tasks"=>
[{"pulp_href"=>"/pulp/api/v3/tasks/bd9ce5bf-417f-4f7f-9e82-23ba62fc9432/",
"pulp_created"=>"2022-03-30T14:37:32.853+00:00",
"state"=>"failed",
"name"=>"pulpcore.app.tasks.orphan.orphan_cleanup",
"logging_cid"=>"60c72b39461e434294cecd403e96ed45",
"started_at"=>"2022-03-30T14:37:35.201+00:00",
"finished_at"=>"2022-03-30T14:37:57.480+00:00",
"error"=>
{"traceback"=>
#<Sequel::SQL::Blob:0x3b0b0 bytes=139 start=" File \"/o" end="form_task\n"> +
#<Sequel::SQL::Blob:0x3b0c4 bytes=35 start=" result" end="**kwargs)\n"> +
#<Sequel::SQL::Blob:0x3b0d8 bytes=132 start=" File \"/o" end="n_cleanup\n"> +
#<Sequel::SQL::Blob:0x3b0ec bytes=15 content=" c.delete()\n"> +
#<Sequel::SQL::Blob:0x3b100 bytes=122 start=" File \"/o" end="in delete\n"> +
#<Sequel::SQL::Blob:0x3b114 bytes=46 start=" delete" end=".delete()\n"> +
#<Sequel::SQL::Blob:0x3b128 bytes=125 start=" File \"/o" end="in delete\n"> +
#<Sequel::SQL::Blob:0x3b13c bytes=30 start=" signal" end="ete.send(\n"> +
#<Sequel::SQL::Blob:0x3b150 bytes=124 start=" File \"/o" end=", in send\n"> +
#<Sequel::SQL::Blob:0x3b164 bytes=13 content=" return [\n"> +
#<Sequel::SQL::Blob:0x3b178 bytes=130 start=" File \"/o" end="listcomp>\n"> +
#<Sequel::SQL::Blob:0x3b18c bytes=62 start=" (recei" end="**named))\n"> +
#<Sequel::SQL::Blob:0x3b1a0 bytes=142 start=" File \"/o" end="_subrepos\n"> +
#<Sequel::SQL::Blob:0x3b1b4 bytes=21 start=" subrep" end=".delete()\n"> +
#<Sequel::SQL::Blob:0x3b1c8 bytes=87 start=" File \"/o" end=" in inner\n"> +
#<Sequel::SQL::Blob:0x3b1dc bytes=31 start=" return" end=", **kwds)\n"> +
#<Sequel::SQL::Blob:0x3b1f0 bytes=123 start=" File \"/o" end="in delete\n"> +
#<Sequel::SQL::Blob:0x3b204 bytes=44 start=" value " end="**kwargs)\n"> +
#<Sequel::SQL::Blob:0x3b218 bytes=121 start=" File \"/o" end="in delete\n"> +
#<Sequel::SQL::Blob:0x3b22c bytes=57 start=" collec" end="_parents)\n"> +
#<Sequel::SQL::Blob:0x3b240 bytes=126 start=" File \"/o" end="n collect\n"> +
#<Sequel::SQL::Blob:0x3b254 bytes=26 start=" raise " end="tedError(\n">,
"description"=>
"(\"Cannot delete some instances of model 'Repository' because they are referenced through protected foreign keys: 'Variant.repository', 'RpmRepository.repository_ptr'.\", {<Variant: AppStream>})"},
"worker"=>"/pulp/api/v3/workers/18de1dd7-32d5-4ff4-88e7-e25c52b96902/",
"child_tasks"=>[],
"progress_reports"=>
[{"message"=>"Clean up orphan Content",
"code"=>"clean-up.content",
"state"=>"running",
"total"=>947,
"done"=>947}],
"created_resources"=>[],
"reserved_resources_record"=>[]}],
"task_groups"=>[],
"poll_attempts"=>{"total"=>16, "failed"=>1}}
Backtrace:
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/pulp3/abstract_async_task.rb:108:in `block in check_for_errors'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/pulp3/abstract_async_task.rb:106:in `each'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/pulp3/abstract_async_task.rb:106:in `check_for_errors'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/pulp3/abstract_async_task.rb:160:in `poll_external_task'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action/polling.rb:100:in `poll_external_task_with_rescue'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action/polling.rb:22:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action/cancellable.rb:14:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/pulp3/abstract_async_task.rb:10:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:582:in `block (3 levels) in execute_run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:32:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/middleware/remote_action.rb:16:in `block in run'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/middleware/remote_action.rb:40:in `block in as_remote_user'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/models/katello/concerns/user_extensions.rb:21:in `cp_config'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/middleware/remote_action.rb:27:in `as_cp_user'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/middleware/remote_action.rb:39:in `as_remote_user'
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.3.0/app/lib/actions/middleware/remote_action.rb:16:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/rails_executor_wrap.rb:14:in `block in run'
/opt/theforeman/tfm/root/usr/share/gems/gems/activesupport-6.0.3.7/lib/active_support/execution_wrapper.rb:88:in `wrap'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/rails_executor_wrap.rb:13:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action/progress.rb:31:in `with_progress_calculation'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action/progress.rb:17:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/load_setting_values.rb:20:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_request_id.rb:15:in `block in run'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_request_id.rb:52:in `restore_current_request_id'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_request_id.rb:15:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_timezone.rb:15:in `block in run'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_timezone.rb:44:in `restore_curent_timezone'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_timezone.rb:15:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_taxonomies.rb:15:in `block in run'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_taxonomies.rb:45:in `restore_current_taxonomies'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_taxonomies.rb:15:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:32:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:27:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware.rb:19:in `pass'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_user.rb:15:in `block in run'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_user.rb:54:in `restore_curent_user'
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-5.2.2/app/lib/actions/middleware/keep_current_user.rb:15:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/stack.rb:23:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/middleware/world.rb:31:in `execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:581:in `block (2 levels) in execute_run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:580:in `catch'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:580:in `block in execute_run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:483:in `block in with_error_handling'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:483:in `catch'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:483:in `with_error_handling'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:575:in `execute_run'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/action.rb:296:in `execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:18:in `block (2 levels) in execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/execution_plan/steps/abstract.rb:167:in `with_meta_calculation'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:17:in `block in execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:32:in `open_action'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:16:in `execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/director.rb:94:in `execute'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/executors/sidekiq/worker_jobs.rb:11:in `block (2 levels) in perform'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/executors.rb:18:in `run_user_code'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/executors/sidekiq/worker_jobs.rb:9:in `block in perform'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/executors/sidekiq/worker_jobs.rb:25:in `with_telemetry'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/executors/sidekiq/worker_jobs.rb:8:in `perform'
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-1.6.3/lib/dynflow/executors/sidekiq/serialization.rb:27:in `perform'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:192:in `execute_job'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:165:in `block (2 levels) in process'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:128:in `block in invoke'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/middleware/chain.rb:133:in `invoke'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:164:in `block in process'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:137:in `block (6 levels) in dispatch'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/job_retry.rb:109:in `local'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:136:in `block (5 levels) in dispatch'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq.rb:37:in `block in <module:Sidekiq>'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:132:in `block (4 levels) in dispatch'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:250:in `stats'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:127:in `block (3 levels) in dispatch'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/job_logger.rb:8:in `call'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:126:in `block (2 levels) in dispatch'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/job_retry.rb:74:in `global'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:125:in `block in dispatch'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/logging.rb:48:in `with_context'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/logging.rb:42:in `with_job_hash_context'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:124:in `dispatch'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:163:in `process'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:83:in `process_one'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/processor.rb:71:in `run'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/util.rb:16:in `watchdog'
/opt/theforeman/tfm/root/usr/share/gems/gems/sidekiq-5.2.7/lib/sidekiq/util.rb:25:in `block in safe_thread'
/opt/theforeman/tfm/root/usr/share/gems/gems/logging-2.3.0/lib/logging/diagnostic_context.rb:474:in `block in create_with_logging_context'
