Sync is stuck in pending

[Lachlan_Musicman]

CentOS 7.3, Katello 3.2, Foreman 1.13.4

And it's cascading - there are four other sync's that haven't finished
because one seems to be in a permanent pending state.

I don't see any obvious cancel/resume function? The cancel link on the sync
status page bounces when clicked but…nothing.

The logs/Dynaflow show no errors, it's just stopped at 71%.

How to solve?

(raw details below)

cheers
L.

{"repository"=>{"id"=>5, "name"=>"epel", "label"=>"epel"},
"product"=>
{"id"=>2,
"name"=>"Extra Packages for Enterprise Linux 7",
"label"=>"Extra_Packages_for_Enterprise_Linux_7_-_x86_64",
"cp_id"=>"1464869283627"},
"provider"=>{"id"=>1, "name"=>"Anonymous"},
"organization"=>
{"id"=>1,
"name"=>"Peter MacCallum Cancer Centre",
"label"=>"Peter_MacCallum_Cancer_Centre"},
"services_checked"=>["pulp", "pulp_auth"],
"id"=>5,
"sync_result"=>
{"contents_changed"=>true,
"pulp_tasks"=>
[{"exception"=>nil,
"task_type"=>"pulp.server.managers.repo.sync.sync",
"href"=>"/pulp/api/v2/tasks/5460a086-d7e8-4510-9549-c75766ec8fc2/",
"task_id"=>"5460a086-d7e8-4510-9549-c75766ec8fc2",
"tags"=>
["pulp:repository:Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel",
"pulp:action:sync"],
"finish_time"=>"2017-02-24T16:07:18Z",
"_ns"=>"task_status",
"start_time"=>"2017-02-24T16:01:23Z",
"traceback"=>nil,
"spawned_tasks"=>
[{"href"=>"/pulp/api/v2/tasks/a9843fac-51de-4fb0-a06d-31c0db9b108e/",
"task_id"=>"a9843fac-51de-4fb0-a06d-31c0db9b108e"}],
"progress_report"=>
{"yum_importer"=>
{"content"=>
{"items_total"=>22,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>
{"rpm_total"=>22,
"rpm_done"=>22,
"drpm_total"=>0,
"drpm_done"=>0},
"size_total"=>16914540,
"size_left"=>0,
"items_left"=>0},
"comps"=>{"state"=>"FINISHED"},
"purge_duplicates"=>{"state"=>"FINISHED"},
"distribution"=>
{"items_total"=>0,
"state"=>"FINISHED",
"error_details"=>[],
"items_left"=>0},
"errata"=>{"state"=>"FINISHED"},
"metadata"=>{"state"=>"FINISHED"}}},
"queue"=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au.dq",
"state"=>"finished",
"worker_name"=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au",
"result"=>
{"result"=>"success",
"importer_id"=>"yum_importer",
"exception"=>nil,
"repo_id"=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel",
"traceback"=>nil,
"started"=>"2017-02-24T16:01:23Z",
"_ns"=>"repo_sync_results",
"completed"=>"2017-02-24T16:07:18Z",
"importer_type_id"=>"yum_importer",
"error_message"=>nil,
"summary"=>
{"content"=>{"state"=>"FINISHED"},
"comps"=>{"state"=>"FINISHED"},
"purge_duplicates"=>{"state"=>"FINISHED"},
"distribution"=>{"state"=>"FINISHED"},
"errata"=>{"state"=>"FINISHED"},
"metadata"=>{"state"=>"FINISHED"}},
"added_count"=>29,
"removed_count"=>-25,
"updated_count"=>14872,
"id"=>"58b05a36bdb1d60a7024bf11",
"details"=>
{"content"=>
{"size_total"=>16914540,
"items_left"=>0,
"items_total"=>22,
"state"=>"FINISHED",
"size_left"=>0,
"details"=>
{"rpm_total"=>22,
"rpm_done"=>22,
"drpm_total"=>0,
"drpm_done"=>0},
"error_details"=>[]},
"comps"=>{"state"=>"FINISHED"},
"purge_duplicates"=>{"state"=>"FINISHED"},
"distribution"=>
{"items_total"=>0,
"state"=>"FINISHED",
"error_details"=>[],
"items_left"=>0},
"errata"=>{"state"=>"FINISHED"},
"metadata"=>{"state"=>"FINISHED"}}},
"error"=>nil,
"_id"=>{"$oid"=>"58b058a1c47a37aeec805a43"},
"id"=>"58b058a1c47a37aeec805a43"},
{"exception"=>nil,
"task_type"=>"pulp.server.managers.repo.publish.publish",
"href"=>"/pulp/api/v2/tasks/a9843fac-51de-4fb0-a06d-31c0db9b108e/",
"task_id"=>"a9843fac-51de-4fb0-a06d-31c0db9b108e",
"tags"=>
["pulp:repository:Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel",
"pulp:action:publish"],
"finish_time"=>"2017-02-24T16:09:45Z",
"ns"=>"task_status",
"start_time"=>"2017-02-24T16:07:21Z",
"traceback"=>nil,
"spawned_tasks"=>[],
"progress_report"=>
{"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-x86_64-epel"=>
[{"num_success"=>1,
"description"=>"Initializing repo metadata",
"step_type"=>"initialize_repo_metadata",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"5a446a30-7f4f-4fdc-beed-6a8fd2889a80",
"num_processed"=>1},
{"num_success"=>0,
"description"=>"Publishing Distribution files",
"step_type"=>"distribution",
"items_total"=>0,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"ed6fbc4a-5bce-40f5-95a0-8138a58ca633",
"num_processed"=>0},
{"num_success"=>11239,
"description"=>"Publishing RPMs",
"step_type"=>"rpms",
"items_total"=>11239,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"5634cc70-459d-4f08-9ca0-6f9783af239c",
"num_processed"=>11239},
{"num_success"=>0,
"description"=>"Publishing Delta RPMs",
"step_type"=>"drpms",
"items_total"=>1,
"state"=>"SKIPPED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"9c172b59-01dd-4913-8d0d-40c367d970ca",
"num_processed"=>0},
{"num_success"=>3448,
"description"=>"Publishing Errata",
"step_type"=>"errata",
"items_total"=>3448,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"76c9b431-7cf9-4edb-a1d0-555cecec03b7",
"num_processed"=>3448},
{"num_success"=>214,
"description"=>"Publishing Comps file",
"step_type"=>"comps",
"items_total"=>214,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"7c3df37c-419c-44c2-adc7-2c74bd537263",
"num_processed"=>214},
{"num_success"=>0,
"description"=>"Publishing Metadata.",
"step_type"=>"metadata",
"items_total"=>0,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"a5f913dd-d33c-4777-aa13-b25e086b4dfb",
"num_processed"=>0},
{"num_success"=>1,
"description"=>"Closing repo metadata",
"step_type"=>"close_repo_metadata",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"0f71d80f-5e6d-4759-8e69-4b0c3bb559f2",
"num_processed"=>1},
{"num_success"=>0,
"description"=>"Generating sqlite files",
"step_type"=>"generate sqlite",
"items_total"=>1,
"state"=>"SKIPPED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"12c2b669-a180-44a5-9b75-e0fffaf3dd49",
"num_processed"=>0},
{"num_success"=>0,
"description"=>"Generating HTML files",
"step_type"=>"repoview",
"items_total"=>1,
"state"=>"SKIPPED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"c3d3f16a-727a-4b7c-839c-45a95a9888a7",
"num_processed"=>0},
{"num_success"=>1,
"description"=>"Publishing files to web",
"step_type"=>"publish_directory",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"cb0246f9-c2ef-49b3-9bd2-da53e71d692a",
"num_processed"=>1},
{"num_success"=>1,
"description"=>"Writing Listings File",
"step_type"=>"initialize_repo_metadata",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"0f12a784-557d-4f4f-8d5b-f34db4dd59ec",
"num_processed"=>1},
{"num_success"=>1,
"description"=>"Writing Listings File",
"step_type"=>"initialize_repo_metadata",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"3dcbf71e-6950-45a1-b834-42ccfc2e8e0f",
"num_processed"=>1}]},
"queue"=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au.dq",
"state"=>"finished",
"worker_name"=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au",
"result"=>
{"result"=>"success",
"exception"=>nil,
"repo_id"=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel",
"started"=>"2017-02-24T16:07:21Z",
"ns"=>"repo_publish_results",
"completed"=>"2017-02-24T16:09:45Z",
"traceback"=>nil,
"distributor_type_id"=>"yum_distributor",
"summary"=>
{"generate sqlite"=>"SKIPPED",
"initialize_repo_metadata"=>"FINISHED",
"rpms"=>"FINISHED",
"close_repo_metadata"=>"FINISHED",
"drpms"=>"SKIPPED",
"comps"=>"FINISHED",
"distribution"=>"FINISHED",
"repoview"=>"SKIPPED",
"publish_directory"=>"FINISHED",
"errata"=>"FINISHED",
"metadata"=>"FINISHED"},
"error_message"=>nil,
"distributor_id"=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel",
"id"=>"58b05ac9bdb1d60a7024c08c",
"details"=>
[{"num_success"=>1,
"description"=>"Initializing repo metadata",
"step_type"=>"initialize_repo_metadata",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"5a446a30-7f4f-4fdc-beed-6a8fd2889a80",
"num_processed"=>1},
{"num_success"=>0,
"description"=>"Publishing Distribution files",
"step_type"=>"distribution",
"items_total"=>0,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"ed6fbc4a-5bce-40f5-95a0-8138a58ca633",
"num_processed"=>0},
{"num_success"=>11239,
"description"=>"Publishing RPMs",
"step_type"=>"rpms",
"items_total"=>11239,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"5634cc70-459d-4f08-9ca0-6f9783af239c",
"num_processed"=>11239},
{"num_success"=>0,
"description"=>"Publishing Delta RPMs",
"step_type"=>"drpms",
"items_total"=>1,
"state"=>"SKIPPED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"9c172b59-01dd-4913-8d0d-40c367d970ca",
"num_processed"=>0},
{"num_success"=>3448,
"description"=>"Publishing Errata",
"step_type"=>"errata",
"items_total"=>3448,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"76c9b431-7cf9-4edb-a1d0-555cecec03b7",
"num_processed"=>3448},
{"num_success"=>214,
"description"=>"Publishing Comps file",
"step_type"=>"comps",
"items_total"=>214,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"7c3df37c-419c-44c2-adc7-2c74bd537263",
"num_processed"=>214},
{"num_success"=>0,
"description"=>"Publishing Metadata.",
"step_type"=>"metadata",
"items_total"=>0,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"a5f913dd-d33c-4777-aa13-b25e086b4dfb",
"num_processed"=>0},
{"num_success"=>1,
"description"=>"Closing repo metadata",
"step_type"=>"close_repo_metadata",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"0f71d80f-5e6d-4759-8e69-4b0c3bb559f2",
"num_processed"=>1},
{"num_success"=>0,
"description"=>"Generating sqlite files",
"step_type"=>"generate sqlite",
"items_total"=>1,
"state"=>"SKIPPED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"12c2b669-a180-44a5-9b75-e0fffaf3dd49",
"num_processed"=>0},
{"num_success"=>0,
"description"=>"Generating HTML files",
"step_type"=>"repoview",
"items_total"=>1,
"state"=>"SKIPPED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"c3d3f16a-727a-4b7c-839c-45a95a9888a7",
"num_processed"=>0},
{"num_success"=>1,
"description"=>"Publishing files to web",
"step_type"=>"publish_directory",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"cb0246f9-c2ef-49b3-9bd2-da53e71d692a",
"num_processed"=>1},
{"num_success"=>1,
"description"=>"Writing Listings File",
"step_type"=>"initialize_repo_metadata",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"0f12a784-557d-4f4f-8d5b-f34db4dd59ec",
"num_processed"=>1},
{"num_success"=>1,
"description"=>"Writing Listings File",
"step_type"=>"initialize_repo_metadata",
"items_total"=>1,
"state"=>"FINISHED",
"error_details"=>[],
"details"=>"",
"num_failures"=>0,
"step_id"=>"3dcbf71e-6950-45a1-b834-42ccfc2e8e0f",
"num_processed"=>1}]},
"error"=>nil,
"_id"=>{"$oid"=>"58b05a36c47a37aeec805b06"},
"id"=>"58b05a36c47a37aeec805b06"}],
"poll_attempts"=>{"total"=>29, "failed"=>0}},
"user_id"=>1,
"contents_changed"=>true,
"current_user_id"=>1}

···

------ The most dangerous phrase in the language is, "We've always done it this way."

• Grace Hopper

[Lachlan_Musicman]

Search turned up this potential solution:

https://access.redhat.com/discussions/2065363

I tried restarting Katello service as noted, without success. I ended up
here

https://access.redhat.com/solutions/1381053

There's an error thrown on the last step:

[root@vmpr-res-utils ~]# foreman-rake katello:reindex --trace
rake aborted!
Don't know how to build task 'katello:reindex'
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task_manager.rb:62:in
[]' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:149:ininvoke_task'
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block (2 levels) in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:ineach'
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:115:inrun_with_threads'
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:100:in
top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:78:inblock in run'
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:176:in
standard_exception_handling' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:75:inrun'
/opt/rh/rh-ruby22/root/usr/bin/rake:33:in `<main>'

That will teach me for trying to solve my own problems.

The problem seems to have persisted - where do I look next?

cheers
L.

···

------ The most dangerous phrase in the language is, "We've always done it this way."

• Grace Hopper

On 27 February 2017 at 09:39, Lachlan Musicman datakid@gmail.com wrote:

CentOS 7.3, Katello 3.2, Foreman 1.13.4

And it’s cascading - there are four other sync’s that haven’t finished
because one seems to be in a permanent pending state.

I don’t see any obvious cancel/resume function? The cancel link on the
sync status page bounces when clicked but…nothing.

The logs/Dynaflow show no errors, it’s just stopped at 71%.

How to solve?

(raw details below)

cheers
L.

{“repository”=>{“id”=>5, “name”=>“epel”, “label”=>“epel”},
“product”=>
{“id”=>2,
“name”=>“Extra Packages for Enterprise Linux 7”,
“label”=>“Extra_Packages_for_Enterprise_Linux_7_-_x86_64”,
“cp_id”=>“1464869283627”},
“provider”=>{“id”=>1, “name”=>“Anonymous”},
“organization”=>
{“id”=>1,
“name”=>“Peter MacCallum Cancer Centre”,
“label”=>“Peter_MacCallum_Cancer_Centre”},
“services_checked”=>[“pulp”, “pulp_auth”],
“id”=>5,
“sync_result”=>
{“contents_changed”=>true,
“pulp_tasks”=>
[{“exception”=>nil,
“task_type”=>“pulp.server.managers.repo.sync.sync”,
“href"=>"/pulp/api/v2/tasks/5460a086-d7e8-4510-9549-c75766ec8fc2/",
“task_id”=>“5460a086-d7e8-4510-9549-c75766ec8fc2”,
“tags”=>
["pulp:repository:Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“pulp:action:sync”],
“finish_time”=>“2017-02-24T16:07:18Z”,
"_ns"=>“task_status”,
“start_time”=>“2017-02-24T16:01:23Z”,
“traceback”=>nil,
“spawned_tasks”=>
[{“href"=>"/pulp/api/v2/tasks/a9843fac-51de-4fb0-a06d-31c0db9b108e/",
“task_id”=>“a9843fac-51de-4fb0-a06d-31c0db9b108e”}],
“progress_report”=>
{“yum_importer”=>
{“content”=>
{“items_total”=>22,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>
{“rpm_total”=>22,
“rpm_done”=>22,
“drpm_total”=>0,
“drpm_done”=>0},
“size_total”=>16914540,
“size_left”=>0,
“items_left”=>0},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>
{“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“items_left”=>0},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}}},
“queue”=>
“reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au.dq”,
“state”=>“finished”,
“worker_name”=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au",
“result”=>
{“result”=>“success”,
“importer_id”=>“yum_importer”,
“exception”=>nil,
“repo_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“traceback”=>nil,
“started”=>“2017-02-24T16:01:23Z”,
"_ns"=>“repo_sync_results”,
“completed”=>“2017-02-24T16:07:18Z”,
“importer_type_id”=>“yum_importer”,
“error_message”=>nil,
“summary”=>
{“content”=>{“state”=>“FINISHED”},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>{“state”=>“FINISHED”},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}},
“added_count”=>29,
“removed_count”=>-25,
“updated_count”=>14872,
“id”=>“58b05a36bdb1d60a7024bf11”,
“details”=>
{“content”=>
{“size_total”=>16914540,
“items_left”=>0,
“items_total”=>22,
“state”=>“FINISHED”,
“size_left”=>0,
“details”=>
{“rpm_total”=>22,
“rpm_done”=>22,
“drpm_total”=>0,
“drpm_done”=>0},
“error_details”=>[]},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>
{“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“items_left”=>0},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}}},
“error”=>nil,
"_id"=>{"$oid"=>“58b058a1c47a37aeec805a43”},
“id”=>“58b058a1c47a37aeec805a43”},
{“exception”=>nil,
“task_type”=>“pulp.server.managers.repo.publish.publish”,
“href"=>"/pulp/api/v2/tasks/a9843fac-51de-4fb0-a06d-31c0db9b108e/",
“task_id”=>“a9843fac-51de-4fb0-a06d-31c0db9b108e”,
“tags”=>
["pulp:repository:Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“pulp:action:publish”],
“finish_time”=>“2017-02-24T16:09:45Z”,
“ns"=>“task_status”,
“start_time”=>“2017-02-24T16:07:21Z”,
“traceback”=>nil,
“spawned_tasks”=>[],
“progress_report”=>
{"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-x86_64-epel"=>
[{“num_success”=>1,
“description”=>“Initializing repo metadata”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5a446a30-7f4f-4fdc-beed-6a8fd2889a80”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Publishing Distribution files”,
“step_type”=>“distribution”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“ed6fbc4a-5bce-40f5-95a0-8138a58ca633”,
“num_processed”=>0},
{“num_success”=>11239,
“description”=>“Publishing RPMs”,
“step_type”=>“rpms”,
“items_total”=>11239,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5634cc70-459d-4f08-9ca0-6f9783af239c”,
“num_processed”=>11239},
{“num_success”=>0,
“description”=>“Publishing Delta RPMs”,
“step_type”=>“drpms”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“9c172b59-01dd-4913-8d0d-40c367d970ca”,
“num_processed”=>0},
{“num_success”=>3448,
“description”=>“Publishing Errata”,
“step_type”=>“errata”,
“items_total”=>3448,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“76c9b431-7cf9-4edb-a1d0-555cecec03b7”,
“num_processed”=>3448},
{“num_success”=>214,
“description”=>“Publishing Comps file”,
“step_type”=>“comps”,
“items_total”=>214,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“7c3df37c-419c-44c2-adc7-2c74bd537263”,
“num_processed”=>214},
{“num_success”=>0,
“description”=>“Publishing Metadata.”,
“step_type”=>“metadata”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“a5f913dd-d33c-4777-aa13-b25e086b4dfb”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Closing repo metadata”,
“step_type”=>“close_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“0f71d80f-5e6d-4759-8e69-4b0c3bb559f2”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Generating sqlite files”,
“step_type”=>“generate sqlite”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“12c2b669-a180-44a5-9b75-e0fffaf3dd49”,
“num_processed”=>0},
{“num_success”=>0,
“description”=>“Generating HTML files”,
“step_type”=>“repoview”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“c3d3f16a-727a-4b7c-839c-45a95a9888a7”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Publishing files to web”,
“step_type”=>“publish_directory”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“cb0246f9-c2ef-49b3-9bd2-da53e71d692a”,
“num_processed”=>1},
{“num_success”=>1,
“description”=>“Writing Listings File”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“0f12a784-557d-4f4f-8d5b-f34db4dd59ec”,
“num_processed”=>1},
{“num_success”=>1,
“description”=>“Writing Listings File”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“3dcbf71e-6950-45a1-b834-42ccfc2e8e0f”,
“num_processed”=>1}]},
“queue”=>
“reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au.dq”,
“state”=>“finished”,
“worker_name”=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au",
“result”=>
{“result”=>“success”,
“exception”=>nil,
“repo_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“started”=>“2017-02-24T16:07:21Z”,
“ns"=>“repo_publish_results”,
“completed”=>“2017-02-24T16:09:45Z”,
“traceback”=>nil,
“distributor_type_id”=>“yum_distributor”,
“summary”=>
{“generate sqlite”=>“SKIPPED”,
“initialize_repo_metadata”=>“FINISHED”,
“rpms”=>“FINISHED”,
“close_repo_metadata”=>“FINISHED”,
“drpms”=>“SKIPPED”,
“comps”=>“FINISHED”,
“distribution”=>“FINISHED”,
“repoview”=>“SKIPPED”,
“publish_directory”=>“FINISHED”,
“errata”=>“FINISHED”,
“metadata”=>“FINISHED”},
“error_message”=>nil,
“distributor_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“id”=>“58b05ac9bdb1d60a7024c08c”,
“details”=>
[{“num_success”=>1,
“description”=>“Initializing repo metadata”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5a446a30-7f4f-4fdc-beed-6a8fd2889a80”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Publishing Distribution files”,
“step_type”=>“distribution”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“ed6fbc4a-5bce-40f5-95a0-8138a58ca633”,
“num_processed”=>0},
{“num_success”=>11239,
“description”=>“Publishing RPMs”,
“step_type”=>“rpms”,
“items_total”=>11239,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5634cc70-459d-4f08-9ca0-6f9783af239c”,
“num_processed”=>11239},
{“num_success”=>0,
“description”=>“Publishing Delta RPMs”,
“step_type”=>“drpms”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“9c172b59-01dd-4913-8d0d-40c367d970ca”,
“num_processed”=>0},
{“num_success”=>3448,
“description”=>“Publishing Errata”,
“step_type”=>“errata”,
“items_total”=>3448,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“76c9b431-7cf9-4edb-a1d0-555cecec03b7”,
“num_processed”=>3448},
{“num_success”=>214,
“description”=>“Publishing Comps file”,
“step_type”=>“comps”,
“items_total”=>214,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“7c3df37c-419c-44c2-adc7-2c74bd537263”,
“num_processed”=>214},
{“num_success”=>0,
“description”=>“Publishing Metadata.”,
“step_type”=>“metadata”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“a5f913dd-d33c-4777-aa13-b25e086b4dfb”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Closing repo metadata”,
“step_type”=>“close_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“0f71d80f-5e6d-4759-8e69-4b0c3bb559f2”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Generating sqlite files”,
“step_type”=>“generate sqlite”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“12c2b669-a180-44a5-9b75-e0fffaf3dd49”,
“num_processed”=>0},
{“num_success”=>0,
“description”=>“Generating HTML files”,
“step_type”=>“repoview”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“c3d3f16a-727a-4b7c-839c-45a95a9888a7”,
“num_processed”=>0},

[sean797]

Try foreman-rake katello:reimport

foreman-rake katello:reindex Has been renamed recently.

···

On Mon, 27 Feb 2017 at 00:49, Lachlan Musicman wrote:

Search turned up this potential solution:

https://access.redhat.com/discussions/2065363

I tried restarting Katello service as noted, without success. I ended up
here

https://access.redhat.com/solutions/1381053

There’s an error thrown on the last step:

[root@vmpr-res-utils ~]# foreman-rake katello:reindex --trace
rake aborted!
Don’t know how to build task ‘katello:reindex’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task_manager.rb:62:in
[]' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:149:ininvoke_task’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block (2 levels) in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:ineach’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:115:inrun_with_threads’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:100:in
top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:78:inblock in run’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:176:in
standard_exception_handling' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:75:inrun’
/opt/rh/rh-ruby22/root/usr/bin/rake:33:in `’

That will teach me for trying to solve my own problems.

The problem seems to have persisted - where do I look next?

cheers
L.

---

The most dangerous phrase in the language is, “We’ve always done it this
way.”

• Grace Hopper

On 27 February 2017 at 09:39, Lachlan Musicman datakid@gmail.com wrote:

CentOS 7.3, Katello 3.2, Foreman 1.13.4

And it’s cascading - there are four other sync’s that haven’t finished
because one seems to be in a permanent pending state.

I don’t see any obvious cancel/resume function? The cancel link on the
sync status page bounces when clicked but…nothing.

The logs/Dynaflow show no errors, it’s just stopped at 71%.

How to solve?

(raw details below)

cheers
L.

{“repository”=>{“id”=>5, “name”=>“epel”, “label”=>“epel”},
“product”=>
{“id”=>2,
“name”=>“Extra Packages for Enterprise Linux 7”,
“label”=>“Extra_Packages_for_Enterprise_Linux_7_-_x86_64”,
“cp_id”=>“1464869283627”},
“provider”=>{“id”=>1, “name”=>“Anonymous”},
“organization”=>
{“id”=>1,
“name”=>“Peter MacCallum Cancer Centre”,
“label”=>“Peter_MacCallum_Cancer_Centre”},
“services_checked”=>[“pulp”, “pulp_auth”],
“id”=>5,
“sync_result”=>
{“contents_changed”=>true,
“pulp_tasks”=>
[{“exception”=>nil,
“task_type”=>“pulp.server.managers.repo.sync.sync”,
“href"=>"/pulp/api/v2/tasks/5460a086-d7e8-4510-9549-c75766ec8fc2/",
“task_id”=>“5460a086-d7e8-4510-9549-c75766ec8fc2”,
“tags”=>
["pulp:repository:Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“pulp:action:sync”],
“finish_time”=>“2017-02-24T16:07:18Z”,
"_ns"=>“task_status”,
“start_time”=>“2017-02-24T16:01:23Z”,
“traceback”=>nil,
“spawned_tasks”=>
[{“href"=>"/pulp/api/v2/tasks/a9843fac-51de-4fb0-a06d-31c0db9b108e/",
“task_id”=>“a9843fac-51de-4fb0-a06d-31c0db9b108e”}],
“progress_report”=>
{“yum_importer”=>
{“content”=>
{“items_total”=>22,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>
{“rpm_total”=>22,
“rpm_done”=>22,
“drpm_total”=>0,
“drpm_done”=>0},
“size_total”=>16914540,
“size_left”=>0,
“items_left”=>0},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>
{“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“items_left”=>0},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}}},
“queue”=>
“reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au.dq”,
“state”=>“finished”,
“worker_name”=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au",
“result”=>
{“result”=>“success”,
“importer_id”=>“yum_importer”,
“exception”=>nil,
“repo_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“traceback”=>nil,
“started”=>“2017-02-24T16:01:23Z”,
"_ns"=>“repo_sync_results”,
“completed”=>“2017-02-24T16:07:18Z”,
“importer_type_id”=>“yum_importer”,
“error_message”=>nil,
“summary”=>
{“content”=>{“state”=>“FINISHED”},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>{“state”=>“FINISHED”},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}},
“added_count”=>29,
“removed_count”=>-25,
“updated_count”=>14872,
“id”=>“58b05a36bdb1d60a7024bf11”,
“details”=>
{“content”=>
{“size_total”=>16914540,
“items_left”=>0,
“items_total”=>22,
“state”=>“FINISHED”,
“size_left”=>0,
“details”=>
{“rpm_total”=>22,
“rpm_done”=>22,
“drpm_total”=>0,
“drpm_done”=>0},
“error_details”=>[]},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>
{“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“items_left”=>0},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}}},
“error”=>nil,
"_id"=>{"$oid"=>“58b058a1c47a37aeec805a43”},
“id”=>“58b058a1c47a37aeec805a43”},
{“exception”=>nil,
“task_type”=>“pulp.server.managers.repo.publish.publish”,
“href"=>"/pulp/api/v2/tasks/a9843fac-51de-4fb0-a06d-31c0db9b108e/",
“task_id”=>“a9843fac-51de-4fb0-a06d-31c0db9b108e”,
“tags”=>
["pulp:repository:Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“pulp:action:publish”],
“finish_time”=>“2017-02-24T16:09:45Z”,
“ns"=>“task_status”,
“start_time”=>“2017-02-24T16:07:21Z”,
“traceback”=>nil,
“spawned_tasks”=>[],
“progress_report”=>
{"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-x86_64-epel"=>
[{“num_success”=>1,
“description”=>“Initializing repo metadata”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5a446a30-7f4f-4fdc-beed-6a8fd2889a80”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Publishing Distribution files”,
“step_type”=>“distribution”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“ed6fbc4a-5bce-40f5-95a0-8138a58ca633”,
“num_processed”=>0},
{“num_success”=>11239,
“description”=>“Publishing RPMs”,
“step_type”=>“rpms”,
“items_total”=>11239,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5634cc70-459d-4f08-9ca0-6f9783af239c”,
“num_processed”=>11239},
{“num_success”=>0,
“description”=>“Publishing Delta RPMs”,
“step_type”=>“drpms”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“9c172b59-01dd-4913-8d0d-40c367d970ca”,
“num_processed”=>0},
{“num_success”=>3448,
“description”=>“Publishing Errata”,
“step_type”=>“errata”,
“items_total”=>3448,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“76c9b431-7cf9-4edb-a1d0-555cecec03b7”,
“num_processed”=>3448},
{“num_success”=>214,
“description”=>“Publishing Comps file”,
“step_type”=>“comps”,
“items_total”=>214,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“7c3df37c-419c-44c2-adc7-2c74bd537263”,
“num_processed”=>214},
{“num_success”=>0,
“description”=>“Publishing Metadata.”,
“step_type”=>“metadata”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“a5f913dd-d33c-4777-aa13-b25e086b4dfb”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Closing repo metadata”,
“step_type”=>“close_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“0f71d80f-5e6d-4759-8e69-4b0c3bb559f2”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Generating sqlite files”,
“step_type”=>“generate sqlite”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“12c2b669-a180-44a5-9b75-e0fffaf3dd49”,
“num_processed”=>0},
{“num_success”=>0,
“description”=>“Generating HTML files”,
“step_type”=>“repoview”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“c3d3f16a-727a-4b7c-839c-45a95a9888a7”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Publishing files to web”,
“step_type”=>“publish_directory”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“cb0246f9-c2ef-49b3-9bd2-da53e71d692a”,
“num_processed”=>1},
{“num_success”=>1,
“description”=>“Writing Listings File”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“0f12a784-557d-4f4f-8d5b-f34db4dd59ec”,
“num_processed”=>1},
{“num_success”=>1,
“description”=>“Writing Listings File”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“3dcbf71e-6950-45a1-b834-42ccfc2e8e0f”,
“num_processed”=>1}]},
“queue”=>
“reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au.dq”,
“state”=>“finished”,
“worker_name”=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au",
“result”=>
{“result”=>“success”,
“exception”=>nil,
“repo_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“started”=>“2017-02-24T16:07:21Z”,
“ns"=>“repo_publish_results”,
“completed”=>“2017-02-24T16:09:45Z”,
“traceback”=>nil,
“distributor_type_id”=>“yum_distributor”,
“summary”=>
{“generate sqlite”=>“SKIPPED”,
“initialize_repo_metadata”=>“FINISHED”,
“rpms”=>“FINISHED”,
“close_repo_metadata”=>“FINISHED”,
“drpms”=>“SKIPPED”,
“comps”=>“FINISHED”,
“distribution”=>“FINISHED”,
“repoview”=>“SKIPPED”,
“publish_directory”=>“FINISHED”,
“errata”=>“FINISHED”,
“metadata”=>“FINISHED”},
“error_message”=>nil,
“distributor_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“id”=>“58b05ac9bdb1d60a7024c08c”,
“details”=>
[{“num_success”=>1,
“description”=>“Initializing repo metadata”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5a446a30-7f4f-4fdc-beed-6a8fd2889a80”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Publishing Distribution files”,
“step_type”=>“distribution”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“ed6fbc4a-5bce-40f5-95a0-8138a58ca633”,
“num_processed”=>0},
{“num_success”=>11239,
“description”=>“Publishing RPMs”,
“step_type”=>“rpms”,
“items_total”=>11239,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5634cc70-459d-4f08-9ca0-6f9783af239c”,
“num_processed”=>11239},
{“num_success”=>0,
“description”=>“Publishing Delta RPMs”,
“step_type”=>“drpms”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“9c172b59-01dd-4913-8d0d-40c367d970ca”,
“num_processed”=>0},
{“num_success”=>3448,
“description”=>“Publishing Errata”,
“step_type”=>“errata”,
“items_total”=>3448,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“76c9b431-7cf9-4edb-a1d0-555cecec03b7”,
“num_processed”=>3448},
{“num_success”=>214,
“description”=>“Publishing Comps file”,
“step_type”=>“comps”,
“items_total”=>214,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“7c3df37c-419c-44c2-adc7-2c74bd537263”,
“num_processed”=>214},
{“num_success”=>0,
“description”=>“Publishing Metadata.”,
“step_type”=>“metadata”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“a5f913dd-d33c-4777-aa13-b25e086b4dfb”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Closing repo metadata”,
“step_type”=>“close_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“0f71d80f-5e6d-4759-8e69-4b0c3bb559f2”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Generating sqlite files”,
“step_type”=>“generate sqlite”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“12c2b669-a180-44a5-9b75-e0fffaf3dd49”,
“num_processed”=>0},
{“num_success”=>0,
“description”=>“Generating HTML files”,
“step_type”=>“repoview”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“c3d3f16a-727a-4b7c-839c-45a95a9888a7”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Publishing files to web”,
“step_type”=>“publish_directory”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“cb0246f9-c2ef-49b3-9bd2-da53e71d692a”,
“num_processed”=>1},
{“num_success”=>1,
“description”=>“Writing Listings File”,
"step_type

[Lachlan_Musicman]

Thankyou - that worked in so much as it didn't throw an error.

I still have the problem of four synchronisations that have stopped but
aren't completing.

By stopped I mean "are not advancing".

As far as I can tell, they are in a running state - which is why the steps
above didn't work, they were only for killing tasks in a paused state.

Label: Actions::Katello::Repository::Sync
Name: Synchronize
Owner: foreman_admin
Execution type: Delayed
Start at: 2017-02-25 03:01:08 +1100
Start before: -
Started at: 2017-02-25 03:01:08 +1100
Ended at:
State: running

So the task started a three days ago and hasn't advanced any more. This job
in particular is blocking the other three from completing according to the
relevant logs.

It looks like I have a suspended
Actions::Pulp::Repository::RegenerateApplicability:

16: Actions::Pulp::Repository::RegenerateApplicability (suspended) [
282330.29s / 1592.86s ]

but I see no documentation on how I might get it out of suspension?

cheers
L.

···

------ The most dangerous phrase in the language is, "We've always done it this way."

• Grace Hopper

On 27 February 2017 at 18:07, Sean O’Keeffe sokeeffe@redhat.com wrote:

Try foreman-rake katello:reimport

foreman-rake katello:reindex Has been renamed recently.

On Mon, 27 Feb 2017 at 00:49, Lachlan Musicman datakid@gmail.com wrote:

Search turned up this potential solution:

https://access.redhat.com/discussions/2065363

I tried restarting Katello service as noted, without success. I ended up
here

https://access.redhat.com/solutions/1381053

There’s an error thrown on the last step:

[root@vmpr-res-utils ~]# foreman-rake katello:reindex --trace
rake aborted!
Don’t know how to build task ‘katello:reindex’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task_manager.rb:62:in
[]' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:149:ininvoke_task’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block (2 levels) in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:ineach’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:115:inrun_with_threads’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:100:in
top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:78:inblock in run’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:176:in
standard_exception_handling' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:75:inrun’
/opt/rh/rh-ruby22/root/usr/bin/rake:33:in `’

That will teach me for trying to solve my own problems.

The problem seems to have persisted - where do I look next?

cheers
L.

---

The most dangerous phrase in the language is, “We’ve always done it this
way.”

• Grace Hopper

On 27 February 2017 at 09:39, Lachlan Musicman datakid@gmail.com wrote:

CentOS 7.3, Katello 3.2, Foreman 1.13.4

And it’s cascading - there are four other sync’s that haven’t finished
because one seems to be in a permanent pending state.

I don’t see any obvious cancel/resume function? The cancel link on the
sync status page bounces when clicked but…nothing.

The logs/Dynaflow show no errors, it’s just stopped at 71%.

How to solve?

(raw details below)

cheers
L.

{“repository”=>{“id”=>5, “name”=>“epel”, “label”=>“epel”},
“product”=>
{“id”=>2,
“name”=>“Extra Packages for Enterprise Linux 7”,
“label”=>“Extra_Packages_for_Enterprise_Linux_7_-_x86_64”,
“cp_id”=>“1464869283627”},
“provider”=>{“id”=>1, “name”=>“Anonymous”},
“organization”=>
{“id”=>1,
“name”=>“Peter MacCallum Cancer Centre”,
“label”=>“Peter_MacCallum_Cancer_Centre”},
“services_checked”=>[“pulp”, “pulp_auth”],
“id”=>5,
“sync_result”=>
{“contents_changed”=>true,
“pulp_tasks”=>
[{“exception”=>nil,
“task_type”=>“pulp.server.managers.repo.sync.sync”,
“href"=>"/pulp/api/v2/tasks/5460a086-d7e8-4510-9549-c75766ec8fc2/",
“task_id”=>“5460a086-d7e8-4510-9549-c75766ec8fc2”,
“tags”=>
["pulp:repository:Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“pulp:action:sync”],
“finish_time”=>“2017-02-24T16:07:18Z”,
"_ns"=>“task_status”,
“start_time”=>“2017-02-24T16:01:23Z”,
“traceback”=>nil,
“spawned_tasks”=>
[{“href"=>"/pulp/api/v2/tasks/a9843fac-51de-4fb0-a06d-31c0db9b108e/",
“task_id”=>“a9843fac-51de-4fb0-a06d-31c0db9b108e”}],
“progress_report”=>
{“yum_importer”=>
{“content”=>
{“items_total”=>22,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>
{“rpm_total”=>22,
“rpm_done”=>22,
“drpm_total”=>0,
“drpm_done”=>0},
“size_total”=>16914540,
“size_left”=>0,
“items_left”=>0},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>
{“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“items_left”=>0},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}}},
“queue”=>
“reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au.dq”,
“state”=>“finished”,
“worker_name”=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au",
“result”=>
{“result”=>“success”,
“importer_id”=>“yum_importer”,
“exception”=>nil,
“repo_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“traceback”=>nil,
“started”=>“2017-02-24T16:01:23Z”,
"_ns"=>“repo_sync_results”,
“completed”=>“2017-02-24T16:07:18Z”,
“importer_type_id”=>“yum_importer”,
“error_message”=>nil,
“summary”=>
{“content”=>{“state”=>“FINISHED”},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>{“state”=>“FINISHED”},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}},
“added_count”=>29,
“removed_count”=>-25,
“updated_count”=>14872,
“id”=>“58b05a36bdb1d60a7024bf11”,
“details”=>
{“content”=>
{“size_total”=>16914540,
“items_left”=>0,
“items_total”=>22,
“state”=>“FINISHED”,
“size_left”=>0,
“details”=>
{“rpm_total”=>22,
“rpm_done”=>22,
“drpm_total”=>0,
“drpm_done”=>0},
“error_details”=>[]},
“comps”=>{“state”=>“FINISHED”},
“purge_duplicates”=>{“state”=>“FINISHED”},
“distribution”=>
{“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“items_left”=>0},
“errata”=>{“state”=>“FINISHED”},
“metadata”=>{“state”=>“FINISHED”}}},
“error”=>nil,
"_id"=>{"$oid"=>“58b058a1c47a37aeec805a43”},
“id”=>“58b058a1c47a37aeec805a43”},
{“exception”=>nil,
“task_type”=>“pulp.server.managers.repo.publish.publish”,
“href"=>"/pulp/api/v2/tasks/a9843fac-51de-4fb0-a06d-31c0db9b108e/",
“task_id”=>“a9843fac-51de-4fb0-a06d-31c0db9b108e”,
“tags”=>
["pulp:repository:Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“pulp:action:publish”],
“finish_time”=>“2017-02-24T16:09:45Z”,
“ns"=>“task_status”,
“start_time”=>“2017-02-24T16:07:21Z”,
“traceback”=>nil,
“spawned_tasks”=>[],
“progress_report”=>
{"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-x86_64-epel"=>
[{“num_success”=>1,
“description”=>“Initializing repo metadata”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5a446a30-7f4f-4fdc-beed-6a8fd2889a80”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Publishing Distribution files”,
“step_type”=>“distribution”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“ed6fbc4a-5bce-40f5-95a0-8138a58ca633”,
“num_processed”=>0},
{“num_success”=>11239,
“description”=>“Publishing RPMs”,
“step_type”=>“rpms”,
“items_total”=>11239,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5634cc70-459d-4f08-9ca0-6f9783af239c”,
“num_processed”=>11239},
{“num_success”=>0,
“description”=>“Publishing Delta RPMs”,
“step_type”=>“drpms”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“9c172b59-01dd-4913-8d0d-40c367d970ca”,
“num_processed”=>0},
{“num_success”=>3448,
“description”=>“Publishing Errata”,
“step_type”=>“errata”,
“items_total”=>3448,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“76c9b431-7cf9-4edb-a1d0-555cecec03b7”,
“num_processed”=>3448},
{“num_success”=>214,
“description”=>“Publishing Comps file”,
“step_type”=>“comps”,
“items_total”=>214,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“7c3df37c-419c-44c2-adc7-2c74bd537263”,
“num_processed”=>214},
{“num_success”=>0,
“description”=>“Publishing Metadata.”,
“step_type”=>“metadata”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“a5f913dd-d33c-4777-aa13-b25e086b4dfb”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Closing repo metadata”,
“step_type”=>“close_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“0f71d80f-5e6d-4759-8e69-4b0c3bb559f2”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Generating sqlite files”,
“step_type”=>“generate sqlite”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“12c2b669-a180-44a5-9b75-e0fffaf3dd49”,
“num_processed”=>0},
{“num_success”=>0,
“description”=>“Generating HTML files”,
“step_type”=>“repoview”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“c3d3f16a-727a-4b7c-839c-45a95a9888a7”,
“num_processed”=>0},
{“num_success”=>1,
“description”=>“Publishing files to web”,
“step_type”=>“publish_directory”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“cb0246f9-c2ef-49b3-9bd2-da53e71d692a”,
“num_processed”=>1},
{“num_success”=>1,
“description”=>“Writing Listings File”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“0f12a784-557d-4f4f-8d5b-f34db4dd59ec”,
“num_processed”=>1},
{“num_success”=>1,
“description”=>“Writing Listings File”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“3dcbf71e-6950-45a1-b834-42ccfc2e8e0f”,
“num_processed”=>1}]},
“queue”=>
“reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au.dq”,
“state”=>“finished”,
“worker_name”=>
"reserved_resource_worker-1@vmpr-res-utils.unix.petermac.org.au",
“result”=>
{“result”=>“success”,
“exception”=>nil,
“repo_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“started”=>“2017-02-24T16:07:21Z”,
“ns"=>“repo_publish_results”,
“completed”=>“2017-02-24T16:09:45Z”,
“traceback”=>nil,
“distributor_type_id”=>“yum_distributor”,
“summary”=>
{“generate sqlite”=>“SKIPPED”,
“initialize_repo_metadata”=>“FINISHED”,
“rpms”=>“FINISHED”,
“close_repo_metadata”=>“FINISHED”,
“drpms”=>“SKIPPED”,
“comps”=>“FINISHED”,
“distribution”=>“FINISHED”,
“repoview”=>“SKIPPED”,
“publish_directory”=>“FINISHED”,
“errata”=>“FINISHED”,
“metadata”=>“FINISHED”},
“error_message”=>nil,
“distributor_id”=>
"Peter_MacCallum_Cancer_Centre-Extra_Packages_for_Enterprise_Linux_7-_x86_64-epel”,
“id”=>“58b05ac9bdb1d60a7024c08c”,
“details”=>
[{“num_success”=>1,
“description”=>“Initializing repo metadata”,
“step_type”=>“initialize_repo_metadata”,
“items_total”=>1,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5a446a30-7f4f-4fdc-beed-6a8fd2889a80”,
“num_processed”=>1},
{“num_success”=>0,
“description”=>“Publishing Distribution files”,
“step_type”=>“distribution”,
“items_total”=>0,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“ed6fbc4a-5bce-40f5-95a0-8138a58ca633”,
“num_processed”=>0},
{“num_success”=>11239,
“description”=>“Publishing RPMs”,
“step_type”=>“rpms”,
“items_total”=>11239,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“5634cc70-459d-4f08-9ca0-6f9783af239c”,
“num_processed”=>11239},
{“num_success”=>0,
“description”=>“Publishing Delta RPMs”,
“step_type”=>“drpms”,
“items_total”=>1,
“state”=>“SKIPPED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=>“9c172b59-01dd-4913-8d0d-40c367d970ca”,
“num_processed”=>0},
{“num_success”=>3448,
“description”=>“Publishing Errata”,
“step_type”=>“errata”,
“items_total”=>3448,
“state”=>“FINISHED”,
“error_details”=>[],
“details”=>"",
“num_failures”=>0,
“step_id”=

[Lachlan_Musicman]

While I know it's deprecated, I've started looking at pulp-admin:
https://www.mankier.com/1/pulp-admin#Tasks

The task - in the GUI, the id, which I think is the Foreman task id, is
dcbef693-85c3-4418-a189-6ebdb262d99f

This returns

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id
dcbef693-85c3-4418-a189-6ebdb262d99f
Enter password:

···

+----------------------------------------------------------------------+ Task Details +----------------------------------------------------------------------+

The following resource(s) could not be found:

dcbef693-85c3-4418-a189-6ebdb262d99f (resource_id)

So this is the wrong id. I go looking in the dynaflow and see this in the
"suspended" task

pulp_task_group:
group_id: 9a8074c1-60ad-460d-80ad-9c6148d739ca

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id
9a8074c1-60ad-460d-80ad-9c6148d739ca
±---------------------------------------------------------------------+
Task Details
±---------------------------------------------------------------------+

The following resource(s) could not be found:

9a8074c1-60ad-460d-80ad-9c6148d739ca (resource_id)

Hmm. That’s not ideal. Maybe I’m wrong - it does say “pulp_task_group” not
"pulp_task":

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks list
±---------------------------------------------------------------------+
Tasks
±---------------------------------------------------------------------+

No tasks found

Hooh boy.

Ok. So I’ve started seeing hints about a “force unlock” in some of the
pulp/katello docs and bug reports, but can’t find anything definitive.

In desperation, I did, as per https://access.redhat.com/solutions/1381053 :

foreman-rake console
ForemanTasks::Task.where(:state => :running).where(:label =>
“Actions::Pulp::Repository::RegenerateApplicability”).destroy_all

And the sync status page in the GUI is returning 404.

And now sync is failing on failed subtasks: RuntimeError: A sub task failed.

pulp-admin returns

Traceback (most recent call last):
File “/usr/bin/pulp-admin”, line 9, in
load_entry_point(‘pulp-client-admin==2.9.3’, ‘console_scripts’,
‘pulp-admin’)()
File “/usr/lib/python2.7/site-packages/pulp/client/admin/init.py”,
line 9, in main
exit_code = launcher.main(read_config(),
exception_handler_class=AdminExceptionHandler)
File “/usr/lib/python2.7/site-packages/pulp/client/launcher.py”, line
115, in main
code = cli.run(args)
File “/usr/lib/python2.7/site-packages/pulp/client/extensions/core.py”,
line 477, in run
code = self.context.exception_handler.handle_exception(e)
File
"/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py",
line 98, in handle_exception
exit_code = handle_func(e)
File
"/usr/lib/python2.7/site-packages/pulp_rpm/extensions/admin/rpm_admin_consumer/bind.py",
line 64, in handle_not_found
ExceptionHandler.handle_not_found(self, e)
File
"/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py",
line 163, in handle_not_found
for resource_type, resource_id in e.extra_data[‘resources’].items():
KeyError: ‘resources’

foreman-rake returns

[root@vmpr-res-utils ~]# foreman-rake katello:delete_orphaned_content
–trace
** Invoke katello:delete_orphaned_content (first_time)
** Invoke environment (first_time)
** Execute environment
** Execute katello:delete_orphaned_content
rake aborted!
RestClient::ResourceNotFound: 404 Resource Not Found
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/abstract_response.rb:74:in
return!' /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:79:inblock in get_response’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:493:in
call' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:493:inprocess_result’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:421:in
block in transmit' /opt/rh/rh-ruby22/root/usr/share/ruby/net/http.rb:853:instart’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:413:in
transmit' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:176:inexecute’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:41:in
execute' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/resource.rb:94:indelete’
/opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:78:in
get_response' /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:66:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/resources/content.rb:89:in
remove_orphans' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/models/katello/glue/pulp/repo.rb:45:indelete_orphaned_content’
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/lib/katello/tasks/delete_orphaned_content.rake:4:in
block (2 levels) in <top (required)>' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:incall’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:in
block in execute' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:ineach’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:in
execute' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:179:inblock in invoke_with_call_chain’
/opt/rh/rh-ruby22/root/usr/share/ruby/monitor.rb:211:in mon_synchronize' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:172:ininvoke_with_call_chain’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:165:in
invoke' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:150:ininvoke_task’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block (2 levels) in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:ineach’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:115:inrun_with_threads’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:100:in
top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:78:inblock in run’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:176:in
standard_exception_handling' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:75:inrun’
/opt/rh/rh-ruby22/root/usr/bin/rake:33:in `'
Tasks: TOP => katello:delete_orphaned_content

The backtrace is:

/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:206:in
check_for_errors!' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:119:inwait_for_sub_plans’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:39:in
initiate' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:16:inblock in run’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in
block in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:intap’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in
assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:inmatch_value’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in
block in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:ineach’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in
match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:inmatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:11:in
run' /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/bulk_action.rb:20:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:506:in
block (3 levels) in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in
pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in
block in run' /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:53:inrestore_curent_user’
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in
run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in
pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:30:in
with_progress_calculation' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:16:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in
call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in
pass' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:inblock in run’
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:22:in
with_locale' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in
call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in
pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:30:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in
call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/world.rb:30:inexecute’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:505:in
block (2 levels) in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:incatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in
block in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in
block in with_error_handling' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:incatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in
with_error_handling' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:499:inexecute_run’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:260:in
execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:9:inblock (2 levels) in execute’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in
call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:inwith_meta_calculation’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:8:in
block in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:22:inopen_action’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:7:in
execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:15:inblock in on_message’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in
block in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:intap’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in
assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:inmatch_value’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in
block in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:ineach’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in
match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:inmatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:12:in
on_message' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/context.rb:46:inon_envelope’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/executes_context.rb:7:in
on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/actor.rb:26:in
on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/awaits.rb:15:in
on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_results.rb:14:in
on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:38:in
process_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:31:inprocess_envelopes?’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:20:in
on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/termination.rb:55:in
on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/removes_child.rb:10:in
on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_results.rb:14:in
on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:161:inprocess_envelope’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:95:in
block in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:118:inblock (2 levels) in schedule_execution’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/synchronization/mri_lockable_object.rb:38:in
`block in synchronize’
/opt/theforeman/tfm/root/usr/s

[Jason_B_Nance]

Are you sure your /etc/httpd/conf.d/pulp.conf isn't empty? This sounds a lot like the failed upgrade symptoms when pulp.conf was getting blown away (or created without an httpd restart).

J

···

> On Feb 27, 2017, at 6:39 PM, Lachlan Musicman wrote: > > While I know it's deprecated, I've started looking at pulp-admin: https://www.mankier.com/1/pulp-admin#Tasks > > The task - in the GUI, the id, which I think is the Foreman task id, is dcbef693-85c3-4418-a189-6ebdb262d99f > > This returns > > [root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id dcbef693-85c3-4418-a189-6ebdb262d99f > Enter password: > +----------------------------------------------------------------------+ > Task Details > +----------------------------------------------------------------------+ > > The following resource(s) could not be found: > > dcbef693-85c3-4418-a189-6ebdb262d99f (resource_id) > > > So this is the wrong id. I go looking in the dynaflow and see this in the "suspended" task > > pulp_task_group: > group_id: 9a8074c1-60ad-460d-80ad-9c6148d739ca > > [root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id 9a8074c1-60ad-460d-80ad-9c6148d739ca > +----------------------------------------------------------------------+ > Task Details > +----------------------------------------------------------------------+ > > The following resource(s) could not be found: > > 9a8074c1-60ad-460d-80ad-9c6148d739ca (resource_id) > > Hmm. That's not ideal. Maybe I'm wrong - it does say "pulp_task_group" not "pulp_task": > > [root@vmpr-res-utils ~]# pulp-admin -u admin tasks list > +----------------------------------------------------------------------+ > Tasks > +----------------------------------------------------------------------+ > > No tasks found > > Hooh boy. > > Ok. So I've started seeing hints about a "force unlock" in some of the pulp/katello docs and bug reports, but can't find anything definitive. > > In desperation, I did, as per https://access.redhat.com/solutions/1381053 : > > foreman-rake console > ForemanTasks::Task.where(:state => :running).where(:label => "Actions::Pulp::Repository::RegenerateApplicability").destroy_all > > And the sync status page in the GUI is returning 404. > > > And now sync is failing on failed subtasks: RuntimeError: A sub task failed. > > pulp-admin returns > > Traceback (most recent call last): > File "/usr/bin/pulp-admin", line 9, in > load_entry_point('pulp-client-admin==2.9.3', 'console_scripts', 'pulp-admin')() > File "/usr/lib/python2.7/site-packages/pulp/client/admin/__init__.py", line 9, in main > exit_code = launcher.main(read_config(), exception_handler_class=AdminExceptionHandler) > File "/usr/lib/python2.7/site-packages/pulp/client/launcher.py", line 115, in main > code = cli.run(args) > File "/usr/lib/python2.7/site-packages/pulp/client/extensions/core.py", line 477, in run > code = self.context.exception_handler.handle_exception(e) > File "/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py", line 98, in handle_exception > exit_code = handle_func(e) > File "/usr/lib/python2.7/site-packages/pulp_rpm/extensions/admin/rpm_admin_consumer/bind.py", line 64, in handle_not_found > ExceptionHandler.handle_not_found(self, e) > File "/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py", line 163, in handle_not_found > for resource_type, resource_id in e.extra_data['resources'].items(): > KeyError: 'resources' > > foreman-rake returns > > [root@vmpr-res-utils ~]# foreman-rake katello:delete_orphaned_content --trace > ** Invoke katello:delete_orphaned_content (first_time) > ** Invoke environment (first_time) > ** Execute environment > ** Execute katello:delete_orphaned_content > rake aborted! > RestClient::ResourceNotFound: 404 Resource Not Found > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/abstract_response.rb:74:in `return!' > /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:79:in `block in get_response' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:493:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:493:in `process_result' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:421:in `block in transmit' > /opt/rh/rh-ruby22/root/usr/share/ruby/net/http.rb:853:in `start' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:413:in `transmit' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:176:in `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:41:in `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/resource.rb:94:in `delete' > /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:78:in `get_response' > /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:66:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/resources/content.rb:89:in `remove_orphans' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/models/katello/glue/pulp/repo.rb:45:in `delete_orphaned_content' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/lib/katello/tasks/delete_orphaned_content.rake:4:in `block (2 levels) in ' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:in `call' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:in `block in execute' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:in `each' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:in `execute' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:179:in `block in invoke_with_call_chain' > /opt/rh/rh-ruby22/root/usr/share/ruby/monitor.rb:211:in `mon_synchronize' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:172:in `invoke_with_call_chain' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:165:in `invoke' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:150:in `invoke_task' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in `block (2 levels) in top_level' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in `each' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in `block in top_level' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:115:in `run_with_threads' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:100:in `top_level' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:78:in `block in run' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:176:in `standard_exception_handling' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:75:in `run' > /opt/rh/rh-ruby22/root/usr/bin/rake:33:in `' > Tasks: TOP => katello:delete_orphaned_content > > > > The backtrace is: > > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:206:in `check_for_errors!' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:119:in `wait_for_sub_plans' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:39:in `initiate' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:16:in `block in run' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in `block in assigns' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `tap' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `assigns' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:in `match_value' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in `block in match?' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `each' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `match?' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:in `match' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:11:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/bulk_action.rb:20:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:506:in `block (3 levels) in execute_run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in `block in run' > /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:53:in `restore_curent_user' > /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:30:in `with_progress_calculation' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:16:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:in `block in run' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:22:in `with_locale' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:30:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/world.rb:30:in `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:505:in `block (2 levels) in execute_run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in `catch' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in `block in execute_run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `block in with_error_handling' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `catch' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `with_error_handling' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:499:in `execute_run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:260:in `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:9:in `block (2 levels) in execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in `with_meta_calculation' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:8:in `block in execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:22:in `open_action' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:7:in `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:15:in `block in on_message' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in `block in assigns' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `tap' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `assigns' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:in `match_value' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in `block in match?' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `each' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `match?' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:in `match' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:12:in `on_message' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/context.rb:46:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/executes_context.rb:7:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/actor.rb:26:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/awaits.rb:15:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_re

[Lachlan_Musicman]

Break or break through.

running

foreman-installer --scenario katello --upgrade

seems to have solved the issue.

···

------ The most dangerous phrase in the language is, "We've always done it this way."

• Grace Hopper

On 28 February 2017 at 11:39, Lachlan Musicman datakid@gmail.com wrote:

While I know it’s deprecated, I’ve started looking at pulp-admin:
https://www.mankier.com/1/pulp-admin#Tasks

The task - in the GUI, the id, which I think is the Foreman task id, is
dcbef693-85c3-4418-a189-6ebdb262d99f

This returns

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id
dcbef693-85c3-4418-a189-6ebdb262d99f
Enter password:
±---------------------------------------------------------------------+
Task Details
±---------------------------------------------------------------------+

The following resource(s) could not be found:

dcbef693-85c3-4418-a189-6ebdb262d99f (resource_id)

So this is the wrong id. I go looking in the dynaflow and see this in the
"suspended" task

pulp_task_group:
group_id: 9a8074c1-60ad-460d-80ad-9c6148d739ca

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id 9a8074c1-60ad-460d-80ad-9c6148d739ca
±---------------------------------------------------------------------+
Task Details
±---------------------------------------------------------------------+

The following resource(s) could not be found:

9a8074c1-60ad-460d-80ad-9c6148d739ca (resource_id)

Hmm. That’s not ideal. Maybe I’m wrong - it does say “pulp_task_group” not
"pulp_task":

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks list
±---------------------------------------------------------------------+
Tasks
±---------------------------------------------------------------------+

No tasks found

Hooh boy.

Ok. So I’ve started seeing hints about a “force unlock” in some of the
pulp/katello docs and bug reports, but can’t find anything definitive.

In desperation, I did, as per https://access.redhat.com/solutions/1381053
:

foreman-rake console
ForemanTasks::Task.where(:state => :running).where(:label =>
“Actions::Pulp::Repository::RegenerateApplicability”).destroy_all

And the sync status page in the GUI is returning 404.

And now sync is failing on failed subtasks: RuntimeError: A sub task
failed.

pulp-admin returns

Traceback (most recent call last):
File “/usr/bin/pulp-admin”, line 9, in
load_entry_point(‘pulp-client-admin==2.9.3’, ‘console_scripts’,
‘pulp-admin’)()
File “/usr/lib/python2.7/site-packages/pulp/client/admin/init.py”,
line 9, in main
exit_code = launcher.main(read_config(), exception_handler_class=
AdminExceptionHandler)
File “/usr/lib/python2.7/site-packages/pulp/client/launcher.py”, line
115, in main
code = cli.run(args)
File “/usr/lib/python2.7/site-packages/pulp/client/extensions/core.py”,
line 477, in run
code = self.context.exception_handler.handle_exception(e)
File “/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py”,
line 98, in handle_exception
exit_code = handle_func(e)
File “/usr/lib/python2.7/site-packages/pulp_rpm/extensions/
admin/rpm_admin_consumer/bind.py”, line 64, in handle_not_found
ExceptionHandler.handle_not_found(self, e)
File “/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py”,
line 163, in handle_not_found
for resource_type, resource_id in e.extra_data[‘resources’].items():
KeyError: ‘resources’

foreman-rake returns

[root@vmpr-res-utils ~]# foreman-rake katello:delete_orphaned_content
–trace
** Invoke katello:delete_orphaned_content (first_time)
** Invoke environment (first_time)
** Execute environment
** Execute katello:delete_orphaned_content
rake aborted!
RestClient::ResourceNotFound: 404 Resource Not Found
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/abstract_response.rb:74:in return!' /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:79:inblock in get_response’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/request.rb:493:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1. 8.0/lib/restclient/request.rb:493:inprocess_result’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/request.rb:421:in block in transmit' /opt/rh/rh-ruby22/root/usr/share/ruby/net/http.rb:853:instart’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/request.rb:413:in transmit' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1. 8.0/lib/restclient/request.rb:176:inexecute’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/request.rb:41:in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1. 8.0/lib/restclient/resource.rb:94:indelete’
/opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:78:in
get_response' /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:66:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.
0/lib/runcible/resources/content.rb:89:in remove_orphans' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1. 1/app/models/katello/glue/pulp/repo.rb:45:indelete_orphaned_content’
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.
1/lib/katello/tasks/delete_orphaned_content.rake:4:in block (2 levels) in <top (required)>' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:incall’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:in
block in execute' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:ineach’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:in
execute' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:179:inblock in invoke_with_call_chain’
/opt/rh/rh-ruby22/root/usr/share/ruby/monitor.rb:211:in mon_synchronize' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:172:ininvoke_with_call_chain’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:165:in
invoke' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:150:ininvoke_task’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block (2 levels) in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:ineach’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:115:inrun_with_threads’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:100:in
top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:78:inblock in run’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:176:in
standard_exception_handling' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:75:inrun’
/opt/rh/rh-ruby22/root/usr/bin/rake:33:in `'
Tasks: TOP => katello:delete_orphaned_content

The backtrace is:

/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:206:in check_for_errors!' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:119:inwait_for_sub_plans’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:39:in initiate' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:16:inblock in run’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in block in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:intap’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:inmatch_value’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in block in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:ineach’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:inmatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:11:in run' /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/bulk_action.rb:20:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:506:in block (3 levels) in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in block in run' /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:53:inrestore_curent_user’
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:30:in with_progress_calculation' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:16:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in pass' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:inblock in run’
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:22:in with_locale' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:30:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/world.rb:30:inexecute’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:505:in block (2 levels) in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:incatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in block in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in block in with_error_handling' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:incatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in with_error_handling' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:499:inexecute_run’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:260:in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:9:inblock (2 levels) in execute’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:inwith_meta_calculation’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:8:in block in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:22:inopen_action’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:7:in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:15:inblock in on_message’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in block in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:intap’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:inmatch_value’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in block in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:ineach’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:inmatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:12:in on_message' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/context.rb:46:inon_envelope’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/executes_context.rb:7:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/actor.rb:26:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/awaits.rb:15:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_results.rb:14:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:38:in process_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:31:inprocess_envelopes?’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:20:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/termination.rb:55:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/removes_child.rb:10:in `on_envelope’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concur

[Lachlan_Musicman]

Yes, I'm sure. Although it's hard to tell now, I think that it was working
before - other repo syncs were working, despite those four failing.

L.

···

------ The most dangerous phrase in the language is, "We've always done it this way."

• Grace Hopper

On 28 February 2017 at 13:45, ‘Jason Nance’ via Foreman users < foreman-users@googlegroups.com> wrote:

Are you sure your /etc/httpd/conf.d/pulp.conf isn’t empty? This sounds a
lot like the failed upgrade symptoms when pulp.conf was getting blown away
(or created without an httpd restart).

J

On Feb 27, 2017, at 6:39 PM, Lachlan Musicman datakid@gmail.com wrote:

While I know it’s deprecated, I’ve started looking at pulp-admin:
https://www.mankier.com/1/pulp-admin#Tasks

The task - in the GUI, the id, which I think is the Foreman task id, is
dcbef693-85c3-4418-a189-6ebdb262d99f

This returns

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id
dcbef693-85c3-4418-a189-6ebdb262d99f
Enter password:
±---------------------------------------------------------------------+
Task Details
±---------------------------------------------------------------------+

The following resource(s) could not be found:

dcbef693-85c3-4418-a189-6ebdb262d99f (resource_id)

So this is the wrong id. I go looking in the dynaflow and see this in the
"suspended" task

pulp_task_group:
group_id: 9a8074c1-60ad-460d-80ad-9c6148d739ca

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id 9a8074c1-60ad-460d-80ad-9c6148d739ca
±---------------------------------------------------------------------+
Task Details
±---------------------------------------------------------------------+

The following resource(s) could not be found:

9a8074c1-60ad-460d-80ad-9c6148d739ca (resource_id)

Hmm. That’s not ideal. Maybe I’m wrong - it does say “pulp_task_group” not
"pulp_task":

[root@vmpr-res-utils ~]# pulp-admin -u admin tasks list
±---------------------------------------------------------------------+
Tasks
±---------------------------------------------------------------------+

No tasks found

Hooh boy.

Ok. So I’ve started seeing hints about a “force unlock” in some of the
pulp/katello docs and bug reports, but can’t find anything definitive.

In desperation, I did, as per https://access.redhat.com/solutions/1381053
:

foreman-rake console
ForemanTasks::Task.where(:state => :running).where(:label =>
“Actions::Pulp::Repository::RegenerateApplicability”).destroy_all

And the sync status page in the GUI is returning 404.

And now sync is failing on failed subtasks: RuntimeError: A sub task
failed.

pulp-admin returns

Traceback (most recent call last):
File “/usr/bin/pulp-admin”, line 9, in
load_entry_point(‘pulp-client-admin==2.9.3’, ‘console_scripts’,
‘pulp-admin’)()
File “/usr/lib/python2.7/site-packages/pulp/client/admin/init.py”,
line 9, in main
exit_code = launcher.main(read_config(), exception_handler_class=
AdminExceptionHandler)
File “/usr/lib/python2.7/site-packages/pulp/client/launcher.py”, line
115, in main
code = cli.run(args)
File “/usr/lib/python2.7/site-packages/pulp/client/extensions/core.py”,
line 477, in run
code = self.context.exception_handler.handle_exception(e)
File “/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py”,
line 98, in handle_exception
exit_code = handle_func(e)
File “/usr/lib/python2.7/site-packages/pulp_rpm/extensions/
admin/rpm_admin_consumer/bind.py”, line 64, in handle_not_found
ExceptionHandler.handle_not_found(self, e)
File “/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py”,
line 163, in handle_not_found
for resource_type, resource_id in e.extra_data[‘resources’].items():
KeyError: ‘resources’

foreman-rake returns

[root@vmpr-res-utils ~]# foreman-rake katello:delete_orphaned_content
–trace
** Invoke katello:delete_orphaned_content (first_time)
** Invoke environment (first_time)
** Execute environment
** Execute katello:delete_orphaned_content
rake aborted!
RestClient::ResourceNotFound: 404 Resource Not Found
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/abstract_response.rb:74:in return!' /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:79:inblock in get_response’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/request.rb:493:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1. 8.0/lib/restclient/request.rb:493:inprocess_result’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/request.rb:421:in block in transmit' /opt/rh/rh-ruby22/root/usr/share/ruby/net/http.rb:853:instart’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/request.rb:413:in transmit' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1. 8.0/lib/restclient/request.rb:176:inexecute’
/opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.
8.0/lib/restclient/request.rb:41:in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1. 8.0/lib/restclient/resource.rb:94:indelete’
/opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:78:in
get_response' /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:66:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.
0/lib/runcible/resources/content.rb:89:in remove_orphans' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1. 1/app/models/katello/glue/pulp/repo.rb:45:indelete_orphaned_content’
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.
1/lib/katello/tasks/delete_orphaned_content.rake:4:in block (2 levels) in <top (required)>' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:incall’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:in
block in execute' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:ineach’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:in
execute' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:179:inblock in invoke_with_call_chain’
/opt/rh/rh-ruby22/root/usr/share/ruby/monitor.rb:211:in mon_synchronize' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:172:ininvoke_with_call_chain’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:165:in
invoke' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:150:ininvoke_task’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block (2 levels) in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:ineach’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in
block in top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:115:inrun_with_threads’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:100:in
top_level' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:78:inblock in run’
/opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:176:in
standard_exception_handling' /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:75:inrun’
/opt/rh/rh-ruby22/root/usr/bin/rake:33:in `'
Tasks: TOP => katello:delete_orphaned_content

The backtrace is:

/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:206:in check_for_errors!' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:119:inwait_for_sub_plans’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:39:in initiate' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:16:inblock in run’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in block in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:intap’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:inmatch_value’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in block in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:ineach’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:inmatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:11:in run' /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/bulk_action.rb:20:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:506:in block (3 levels) in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in block in run' /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:53:inrestore_curent_user’
/opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:30:in with_progress_calculation' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:16:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in pass' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:inblock in run’
/opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:22:in with_locale' /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in pass' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:30:inrun’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/world.rb:30:inexecute’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:505:in block (2 levels) in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:incatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in block in execute_run' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:incall’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in block in with_error_handling' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:incatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in with_error_handling' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:499:inexecute_run’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:260:in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:9:inblock (2 levels) in execute’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in call' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:inwith_meta_calculation’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:8:in block in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:22:inopen_action’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:7:in execute' /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:15:inblock in on_message’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in block in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:intap’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in assigns' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:inmatch_value’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in block in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:ineach’
/opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in match?' /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:inmatch’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:12:in on_message' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/context.rb:46:inon_envelope’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/executes_context.rb:7:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/actor.rb:26:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/awaits.rb:15:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_results.rb:14:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:38:in process_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:31:inprocess_envelopes?’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:20:in on_envelope' /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:inpass’
/opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/terminatio

[pascalp]

I have this problem (stuck repo sync, CentOS 7.3, Katello 3.2, Foreman
1.13.4) every now and then.
So when one sync (and all the others scheduled for the same worker process)
hangs I do the following:

foreman-rake foreman_tasks:cleanup
TASK_SEARCH='label="Actions::Katello::Repository::Sync"' STATES="paused"
foreman-rake katello:delete_orphaned_content --trace
foreman-rake katello:reimport --trace

Not sure if the last two steps are needed but this fixes it for me.
Hope this helps.

···

On Tuesday, February 28, 2017 at 5:05:29 AM UTC+1, Lachlan Musicman wrote: > > Yes, I'm sure. Although it's hard to tell now, I think that it was working > before - other repo syncs were working, despite those four failing. > > L. > > ------ > The most dangerous phrase in the language is, "We've always done it this > way." > > - Grace Hopper > > On 28 February 2017 at 13:45, 'Jason Nance' via Foreman users < > forema...@googlegroups.com > wrote: > > Are you sure your /etc/httpd/conf.d/pulp.conf isn't empty? This sounds a > lot like the failed upgrade symptoms when pulp.conf was getting blown away > (or created without an httpd restart). > > J > > > On Feb 27, 2017, at 6:39 PM, Lachlan Musicman > wrote: > > While I know it's deprecated, I've started looking at pulp-admin: > https://www.mankier.com/1/pulp-admin#Tasks > > The task - in the GUI, the id, which I think is the Foreman task id, is > dcbef693-85c3-4418-a189-6ebdb262d99f > > This returns > > [root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id > dcbef693-85c3-4418-a189-6ebdb262d99f > Enter password: > +----------------------------------------------------------------------+ > Task Details > +----------------------------------------------------------------------+ > > The following resource(s) could not be found: > > dcbef693-85c3-4418-a189-6ebdb262d99f (resource_id) > > > So this is the wrong id. I go looking in the dynaflow and see this in the > "suspended" task > > pulp_task_group: > group_id: 9a8074c1-60ad-460d-80ad-9c6148d739ca > > [root@vmpr-res-utils ~]# pulp-admin -u admin tasks details --task-id 9a8074c1-60ad-460d-80ad-9c6148d739ca > +----------------------------------------------------------------------+ > Task Details > +----------------------------------------------------------------------+ > > The following resource(s) could not be found: > > 9a8074c1-60ad-460d-80ad-9c6148d739ca (resource_id) > > Hmm. That's not ideal. Maybe I'm wrong - it does say "pulp_task_group" not > "pulp_task": > > [root@vmpr-res-utils ~]# pulp-admin -u admin tasks list > +----------------------------------------------------------------------+ > Tasks > +----------------------------------------------------------------------+ > > No tasks found > > Hooh boy. > > Ok. So I've started seeing hints about a "force unlock" in some of the > pulp/katello docs and bug reports, but can't find anything definitive. > > In desperation, I did, as per https://access.redhat.com/solutions/1381053 > : > > foreman-rake console > ForemanTasks::Task.where(:state => :running).where(:label => > "Actions::Pulp::Repository::RegenerateApplicability").destroy_all > > And the sync status page in the GUI is returning 404. > > > And now sync is failing on failed subtasks: RuntimeError: A sub task > failed. > > pulp-admin returns > > Traceback (most recent call last): > File "/usr/bin/pulp-admin", line 9, in > load_entry_point('pulp-client-admin==2.9.3', 'console_scripts', > 'pulp-admin')() > File "/usr/lib/python2.7/site-packages/pulp/client/admin/__init__.py", > line 9, in main > exit_code = launcher.main(read_config(), > exception_handler_class=AdminExceptionHandler) > File "/usr/lib/python2.7/site-packages/pulp/client/launcher.py", line > 115, in main > code = cli.run(args) > File "/usr/lib/python2.7/site-packages/pulp/client/extensions/core.py", > line 477, in run > code = self.context.exception_handler.handle_exception(e) > File > "/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py", > line 98, in handle_exception > exit_code = handle_func(e) > File > "/usr/lib/python2.7/site-packages/pulp_rpm/extensions/admin/rpm_admin_consumer/bind.py", > line 64, in handle_not_found > ExceptionHandler.handle_not_found(self, e) > File > "/usr/lib/python2.7/site-packages/pulp/client/extensions/exceptions.py", > line 163, in handle_not_found > for resource_type, resource_id in e.extra_data['resources'].items(): > KeyError: 'resources' > > foreman-rake returns > > [root@vmpr-res-utils ~]# foreman-rake katello:delete_orphaned_content > --trace > ** Invoke katello:delete_orphaned_content (first_time) > ** Invoke environment (first_time) > ** Execute environment > ** Execute katello:delete_orphaned_content > rake aborted! > RestClient::ResourceNotFound: 404 Resource Not Found > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/abstract_response.rb:74:in > `return!' > /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:79:in > `block in get_response' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:493:in > `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:493:in > `process_result' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:421:in > `block in transmit' > /opt/rh/rh-ruby22/root/usr/share/ruby/net/http.rb:853:in `start' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:413:in > `transmit' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:176:in > `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/request.rb:41:in > `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/rest-client-1.8.0/lib/restclient/resource.rb:94:in > `delete' > /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:78:in > `get_response' > /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/base.rb:66:in > `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/runcible-1.9.0/lib/runcible/resources/content.rb:89:in > `remove_orphans' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/models/katello/glue/pulp/repo.rb:45:in > `delete_orphaned_content' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/lib/katello/tasks/delete_orphaned_content.rake:4:in > `block (2 levels) in ' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:in > `call' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:240:in > `block in execute' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:in > `each' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:235:in > `execute' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:179:in > `block in invoke_with_call_chain' > /opt/rh/rh-ruby22/root/usr/share/ruby/monitor.rb:211:in `mon_synchronize' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:172:in > `invoke_with_call_chain' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/task.rb:165:in > `invoke' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:150:in > `invoke_task' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in > `block (2 levels) in top_level' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in > `each' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:106:in > `block in top_level' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:115:in > `run_with_threads' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:100:in > `top_level' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:78:in > `block in run' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:176:in > `standard_exception_handling' > /opt/rh/rh-ruby22/root/usr/share/gems/gems/rake-10.4.2/lib/rake/application.rb:75:in > `run' > /opt/rh/rh-ruby22/root/usr/bin/rake:33:in `' > Tasks: TOP => katello:delete_orphaned_content > > > > The backtrace is: > > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:206:in `check_for_errors!' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:119:in `wait_for_sub_plans' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:39:in `initiate' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:16:in `block in run' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in `block in assigns' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `tap' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `assigns' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:in `match_value' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in `block in match?' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `each' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `match?' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:in `match' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/with_sub_plans.rb:11:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/bulk_action.rb:20:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:506:in `block (3 levels) in execute_run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in `block in run' > /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:53:in `restore_curent_user' > /opt/theforeman/tfm/root/usr/share/gems/gems/foreman-tasks-0.8.2/app/lib/actions/middleware/keep_current_user.rb:30:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:30:in `with_progress_calculation' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action/progress.rb:16:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:in `block in run' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:22:in `with_locale' > /opt/theforeman/tfm/root/usr/share/gems/gems/katello-3.2.1.1/app/lib/actions/middleware/keep_locale.rb:11:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:26:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:17:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware.rb:30:in `run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/stack.rb:22:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/middleware/world.rb:30:in `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:505:in `block (2 levels) in execute_run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in `catch' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:504:in `block in execute_run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `block in with_error_handling' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `catch' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:419:in `with_error_handling' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:499:in `execute_run' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/action.rb:260:in `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:9:in `block (2 levels) in execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in `call' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract.rb:155:in `with_meta_calculation' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:8:in `block in execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:22:in `open_action' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/execution_plan/steps/abstract_flow_step.rb:7:in `execute' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:15:in `block in on_message' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:74:in `block in assigns' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `tap' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matchers/abstract.rb:73:in `assigns' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:56:in `match_value' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:36:in `block in match?' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `each' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:35:in `match?' > /opt/theforeman/tfm/root/usr/share/gems/gems/algebrick-0.7.3/lib/algebrick/matching.rb:23:in `match' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/executors/parallel/worker.rb:12:in `on_message' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/context.rb:46:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/executes_context.rb:7:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/dynflow-0.8.15/lib/dynflow/actor.rb:26:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/awaits.rb:15:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_results.rb:14:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:38:in `process_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:31:in `process_envelopes?' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/buffer.rb:20:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/termination.rb:55:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/removes_child.rb:10:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/abstract.rb:25:in `pass' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/behaviour/sets_results.rb:14:in `on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:161:in `process_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:95:in `block in on_envelope' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:118:in `block (2 levels) in schedule_execution' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/synchronization/mri_lockable_object.rb:38:in `block in synchronize' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/synchronization/mri_lockable_object.rb:38:in `synchronize' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/concurrent/synchronization/mri_lockable_object.rb:38:in `synchronize' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-edge-0.2.0/lib/concurrent/actor/core.rb:115:in `block in schedule_execution' > /opt/theforeman/tfm/root/usr/share/gems/gems/concurrent-ruby-1.0.0/lib/