Undefined method pulp3_skip_publication error after upgrade

cave · October 22, 2021, 6:04am

 ::Katello::RepositoryTypeManager.find('yum')
=> nil

::Katello::RepositoryTypeManager.find_defined('yum')
=> #<Katello::RepositoryType:0x000000000c4dfc68 @id=:yum, @allow_creation_by_user=true, @unique_content_per_repo=false, @content_types=[#<Katello::RepositoryType::ContentType:0x000000000af93850 @model_class=Katello::Rpm(id: integer, pulp_id: string, created_at: datetime, updated_at: datetime, name: string, version: string, release: string, arch: string, epoch: string, filename: string, sourcerpm: string, checksum: string, version_sortable: string, release_sortable: string, summary: string, nvra: string, modular: boolean, migrated_pulp3_href: string, evr: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=1, @pulp2_service_class=Katello::Pulp::Rpm, @pulp3_service_class=Katello::Pulp3::Rpm, @index=true, @index_on_pulp3=true, @uploadable=true, @removable=true>, #<Katello::RepositoryType::ContentType:0x000000000af7b728 @model_class=Katello::ModuleStream(id: integer, name: string, pulp_id: string, version: string, context: string, stream: string, arch: string, created_at: datetime, updated_at: datetime, description: text, summary: text, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=2, @pulp2_service_class=Katello::Pulp::ModuleStream, @pulp3_service_class=Katello::Pulp3::ModuleStream, @index=true, @index_on_pulp3=true, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x000000000c56e058 @model_class=Katello::Erratum(id: integer, pulp_id: string, errata_id: string, created_at: datetime, updated_at: datetime, issued: date, updated: date, errata_type: string, severity: string, title: string, solution: text, description: text, summary: text, reboot_suggested: boolean, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=3, @pulp2_service_class=Katello::Pulp::Erratum, @pulp3_service_class=Katello::Pulp3::Erratum, @index=true, @index_on_pulp3=true, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x000000000af2f030 @model_class=Katello::PackageGroup(id: integer, name: string, pulp_id: string, description: string, created_at: datetime, updated_at: datetime, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=99, @pulp2_service_class=Katello::Pulp::PackageGroup, @pulp3_service_class=Katello::Pulp3::PackageGroup, @index=true, @index_on_pulp3=true, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x000000000c5bd888 @model_class=Katello::YumMetadataFile(id: integer, pulp_id: string, checksum: string, name: string, repository_id: integer, created_at: datetime, updated_at: datetime, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=99, @pulp2_service_class=Katello::Pulp::YumMetadataFile, @pulp3_service_class=Katello::Pulp3::YumMetadataFile, @index=true, @index_on_pulp3=false, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x000000000c6094e0 @model_class=Katello::Srpm(id: integer, pulp_id: string, created_at: datetime, updated_at: datetime, name: string, version: string, release: string, arch: string, epoch: string, filename: string, checksum: string, version_sortable: string, release_sortable: string, summary: string, nvra: string, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=99, @pulp2_service_class=Katello::Pulp::Srpm, @pulp3_service_class=Katello::Pulp3::Srpm, @index=true, @index_on_pulp3=true, @uploadable=true, @removable=true>, #<Katello::RepositoryType::ContentType:0x000000000c60e1e8 @model_class=Katello::Distribution, @priority=4, @pulp2_service_class=Katello::Pulp::Distribution, @pulp3_service_class=Katello::Pulp3::Distribution, @index=false, @index_on_pulp3=true, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x000000000aecdc90 @model_class=Katello::PackageCategory, @priority=4, @pulp2_service_class=Katello::Pulp::PackageCategory, @pulp3_service_class=nil, @index=false, @index_on_pulp3=true, @uploadable=false, @removable=false>], @generic_remote_options=[], @service_class=Katello::Pulp::Repository::Yum, @pulp3_service_class=Katello::Pulp3::Repository::Yum, @pulp3_api_class=Katello::Pulp3::Api::Yum, @pulp3_plugin="rpm", @metadata_publish_matching_check=true, @default_managed_content_type_class=Katello::Rpm(id: integer, pulp_id: string, created_at: datetime, updated_at: datetime, name: string, version: string, release: string, arch: string, epoch: string, filename: string, sourcerpm: string, checksum: string, version_sortable: string, release_sortable: string, summary: string, nvra: string, modular: boolean, migrated_pulp3_href: string, evr: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @index_additional_data_proc=#<Proc:0x000000000aecdc18 /opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.2.0.1.rc3/lib/katello/repository_types/yum.rb:39>>

::Katello::RepositoryTypeManager.pulp3_plugin_installed?('yum')
=> nil

::Katello::RepositoryTypeManager.instance_variable_get(:@pulp_primary)
=> #<SmartProxy id: 2, name: "c7foremanproxy.fqdn", url: "https://c7foremanproxy.fqdn:8443", created_at: "2021-10-11 14:44:47", updated_at: "2021-10-11 14:44:47", pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmRkPrQYZfCN...", expired_logs: "0", puppet_path: "", download_policy: "on_demand">

::Katello::RepositoryTypeManager.instance_variable_get(:@pulp_primary).capabilities('Pulpcore')
=> nil

Kind regards

obsti · October 22, 2021, 6:51am

Hi @iballou,
here are my results:

irb(main):001:0> ::Katello::RepositoryTypeManager.find('yum')
=> nil

irb(main):002:0> ::Katello::RepositoryTypeManager.find_defined('yum')
=> #<Katello::RepositoryType:0x0000000009934208 @id=:yum, @allow_creation_by_user=true, @unique_content_per_repo=false, @content_types=[#<Katello::RepositoryType::ContentType:0x00000000096fe4e8 @model_class=Katello::Rpm(id: integer, pulp_id: string, created_at: datetime, updated_at: datetime, name: string, version: string, release: string, arch: string, epoch: string, filename: string, sourcerpm: string, checksum: string, version_sortable: string, release_sortable: string, summary: string, nvra: string, modular: boolean, migrated_pulp3_href: string, evr: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=1, @pulp2_service_class=Katello::Pulp::Rpm, @pulp3_service_class=Katello::Pulp3::Rpm, @index=true, @index_on_pulp3=true, @uploadable=true, @removable=true>, #<Katello::RepositoryType::ContentType:0x00000000097a67d8 @model_class=Katello::ModuleStream(id: integer, name: string, pulp_id: string, version: string, context: string, stream: string, arch: string, created_at: datetime, updated_at: datetime, description: text, summary: text, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=2, @pulp2_service_class=Katello::Pulp::ModuleStream, @pulp3_service_class=Katello::Pulp3::ModuleStream, @index=true, @index_on_pulp3=true, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x000000000973fb00 @model_class=Katello::Erratum(id: integer, pulp_id: string, errata_id: string, created_at: datetime, updated_at: datetime, issued: date, updated: date, errata_type: string, severity: string, title: string, solution: text, description: text, summary: text, reboot_suggested: boolean, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=3, @pulp2_service_class=Katello::Pulp::Erratum, @pulp3_service_class=Katello::Pulp3::Erratum, @index=true, @index_on_pulp3=true, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x000000000968bd58 @model_class=Katello::PackageGroup(id: integer, name: string, pulp_id: string, description: string, created_at: datetime, updated_at: datetime, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=99, @pulp2_service_class=Katello::Pulp::PackageGroup, @pulp3_service_class=Katello::Pulp3::PackageGroup, @index=true, @index_on_pulp3=true, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x0000000009681e20 @model_class=Katello::YumMetadataFile(id: integer, pulp_id: string, checksum: string, name: string, repository_id: integer, created_at: datetime, updated_at: datetime, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=99, @pulp2_service_class=Katello::Pulp::YumMetadataFile, @pulp3_service_class=Katello::Pulp3::YumMetadataFile, @index=true, @index_on_pulp3=false, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x00000000094202d0 @model_class=Katello::Srpm(id: integer, pulp_id: string, created_at: datetime, updated_at: datetime, name: string, version: string, release: string, arch: string, epoch: string, filename: string, checksum: string, version_sortable: string, release_sortable: string, summary: string, nvra: string, migrated_pulp3_href: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @priority=99, @pulp2_service_class=Katello::Pulp::Srpm, @pulp3_service_class=Katello::Pulp3::Srpm, @index=true, @index_on_pulp3=true, @uploadable=true, @removable=true>, #<Katello::RepositoryType::ContentType:0x0000000009400318 @model_class=Katello::Distribution, @priority=4, @pulp2_service_class=Katello::Pulp::Distribution, @pulp3_service_class=Katello::Pulp3::Distribution, @index=false, @index_on_pulp3=true, @uploadable=false, @removable=false>, #<Katello::RepositoryType::ContentType:0x00000000093ceca0 @model_class=Katello::PackageCategory, @priority=4, @pulp2_service_class=Katello::Pulp::PackageCategory, @pulp3_service_class=nil, @index=false, @index_on_pulp3=true, @uploadable=false, @removable=false>], @generic_remote_options=[], @pulp3_service_class=Katello::Pulp3::Repository::Yum, @pulp3_api_class=Katello::Pulp3::Api::Yum, @pulp3_plugin="rpm", @service_class=Katello::Pulp::Repository::Yum, @metadata_publish_matching_check=true, @default_managed_content_type_class=Katello::Rpm(id: integer, pulp_id: string, created_at: datetime, updated_at: datetime, name: string, version: string, release: string, arch: string, epoch: string, filename: string, sourcerpm: string, checksum: string, version_sortable: string, release_sortable: string, summary: string, nvra: string, modular: boolean, migrated_pulp3_href: string, evr: string, missing_from_migration: boolean, ignore_missing_from_migration: boolean), @index_additional_data_proc=#<Proc:0x00000000093cec28 /opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.2.0.1/lib/katello/repository_types/yum.rb:39>>

irb(main):003:0> ::Katello::RepositoryTypeManager.pulp3_plugin_installed?('yum')
=> nil

irb(main):004:0> ::Katello::RepositoryTypeManager.instance_variable_get(:@pulp_primary)
=> #<SmartProxy id: 2, name: "foremanproxy.fqdn", url: "https://foremanproxy.fqdn:9090", created_at: "2017-02-03 09:14:29", updated_at: "2020-05-12 13:18:46", expired_logs: "0", puppet_path: "/etc/puppetlabs/code/environments", download_policy: "on_demand", pubkey: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3vvxmOsOui/...">

irb(main):005:0> ::Katello::RepositoryTypeManager.instance_variable_get(:@pulp_primary).capabilities('Pulpcore')
=> nil

Justin_Sherrill · October 25, 2021, 12:49pm

I think i might see where the issue is (and also explains why we couldn’t reproduce it locally), can you run one more command in the console:

::SmartProxy.unscoped.select { |proxy| !proxy.setting('Pulpcore', 'mirror') }.count

This should confirm the problem, if it returns more than 1, it should be a one line fix.

Justin_Sherrill · October 25, 2021, 1:39pm

I actually went ahead and opened a pr here: https://github.com/Katello/katello/pull/9743

if you want to try to change the code locally, simply restart all the services afterwards (foreman-maintain service restart)

Let me know if that helps and we’ll work on getting a 4.2.1 out ASAP

obsti · October 25, 2021, 2:00pm

I can confirm that it retruns 2 as value:

# foreman-rake console
Loading production environment (Rails 6.0.3.7)
irb(main):001:0> ::SmartProxy.unscoped.select { |proxy| !proxy.setting('Pulpcore', 'mirror') }.count
=> 2

I will try the fix as soon as possible

obsti · October 25, 2021, 2:31pm

After implementing pull request 9743 some services did not startup again.

/ All services displayed                                              [FAIL]
Some services are not running (dynflow-sidekiq@orchestrator, foreman, dynflow-sidekiq@worker-1, dynflow-sidekiq@worker-hosts-queue-1)

See journalctl log:

Oct 25 16:27:34  foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/railties-6.0.3.7/lib/rails/commands/server/server_command.rb:39:in `start'
Oct 25 16:27:34 t foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/railties-6.0.3.7/lib/rails/commands/server/server_command.rb:147:in `block in perform'
Oct 25 16:27:34  foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/railties-6.0.3.7/lib/rails/commands/server/server_command.rb:138:in `tap'
Oct 25 16:27:34  foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/railties-6.0.3.7/lib/rails/commands/server/server_command.rb:138:in `perform'
Oct 25 16:27:34 t foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/thor-1.0.1/lib/thor/command.rb:27:in `run'
Oct 25 16:27:34  foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/thor-1.0.1/lib/thor/invocation.rb:127:in `invoke_command'
Oct 25 16:27:34  foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/thor-1.0.1/lib/thor.rb:392:in `dispatch'
Oct 25 16:27:34  foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/railties-6.0.3.7/lib/rails/command/base.rb:69:in `perform'
Oct 25 16:27:34  foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/railties-6.0.3.7/lib/rails/command.rb:46:in `invoke'
Oct 25 16:27:34  foreman[26210]: from /opt/theforeman/tfm/root/usr/share/gems/gems/railties-6.0.3.7/lib/rails/commands.rb:18:in `<top (required)>'
Oct 25 16:27:34  foreman[26210]: from /opt/rh/rh-ruby27/root/usr/share/rubygems/rubygems/core_ext/kernel_require.rb:158:in `require'
Oct 25 16:27:34  foreman[26210]: from /opt/rh/rh-ruby27/root/usr/share/rubygems/rubygems/core_ext/kernel_require.rb:158:in `rescue in require'
Oct 25 16:27:34  foreman[26210]: from /opt/rh/rh-ruby27/root/usr/share/rubygems/rubygems/core_ext/kernel_require.rb:147:in `require'
Oct 25 16:27:34  foreman[26210]: from /usr/share/foreman/bin/rails:9:in `<main>'
Oct 25 16:27:34 t foreman[26210]: /opt/rh/rh-ruby27/root/usr/share/rubygems/rubygems/core_ext/kernel_require.rb:83:in `require': cannot load such file -- rails/commands (LoadError)
Oct 25 16:27:34  foreman[26210]: from /opt/rh/rh-ruby27/root/usr/share/rubygems/rubygems/core_ext/kernel_require.rb:83:in `require'
Oct 25 16:27:34  foreman[26210]: from /usr/share/foreman/bin/rails:9:in `<main>'
Oct 25 16:27:34 t systemd[1]: foreman.service: main process exited, code=exited, status=1/FAILURE
Oct 25 16:27:34  systemd[1]: Failed to start Foreman.

After rolling back the change all services are starting again without any failure.

Justin_Sherrill · October 25, 2021, 2:55pm

I see the problem, will look into it. Thanks!

Justin_Sherrill · October 25, 2021, 8:19pm

Can you please try the updated change: https://github.com/Katello/katello/pull/9743

The tests all passed with that one. I tested it locally on a production install as well to make sure it still worked:

wget https://github.com/Katello/katello/pull/9743.patch
patch -p1 -d  /opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.2.0.1/ < 9743.patch 
systemctl restart foreman dynflow*

obsti · October 26, 2021, 5:53am

I implemented the patch. Now Katello is starting again.

But the repo syncs still are still not working.
The issue is still there:

cave · October 26, 2021, 6:44am

Hi @Justin_Sherrill

I patched (katello-4.2.0.1.rc3) and restarted:

# patch -p1 -d  /opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.2.0.1.rc3/ < 9743.patch
patching file app/services/katello/repository_type_manager.rb
patching file lib/katello/engine.rb
patching file lib/katello/plugin.rb
# systemctl restart foreman dynflow*

But no services available:

Justin_Sherrill · October 26, 2021, 5:28pm

After some db mangling, i’ve been able to reproduce this!!! working on a fix that i confirm works

Justin_Sherrill · October 27, 2021, 12:52pm

Okay, i’ve got a 2nd patch for you that fixes it on my reproducer. Its a bit longer than i was hoping, and i had to make a katello 4.2-specific version of it. The 4.2 pr is here: https://github.com/Katello/katello/pull/9747/ and i split out the test fixes for easier applying:

Note, i would recomend reinstalling the tfm-rubygem-katello package to remove any previous patches:

yum reinstall -y tfm-rubygem-katello 
wget https://github.com/Katello/katello/pull/9747/commits/fff3e1aa2cc1e9fd582783327a20fa06a09af1be.patch
patch -p1 -d /opt/theforeman/tfm/root/usr/share/gems/gems/katello-4.2.0.1/ < fff3e1aa2cc1e9fd582783327a20fa06a09af1be.patch 
systemctl restart foreman  dynflow*

cave · October 28, 2021, 6:40am

Hi @Justin_Sherrill ,

The reinstall and patch worked for me.
Sync and publish are working again.

Thanks!

obsti · October 28, 2021, 12:09pm

The Patch also sovles the issue for me.
Content Types are back again.

But now I´m hitting another issue. My repo syncs are now crashing with a different error message:

Here is the error in the pulp.log

Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: pulp [3f22f7cd-bbfd-4eb2-99b3-2f5a36a89d38]: pulpcore.tasking.pulpcore_worker:INFO: Task 5901ccb8-faaa-4ad7-b688-a238ddfba996 failed ([Errno 2] No such file or directory: '/var/lib/pulp/tmp/2759@katello.fqdn/tmpvv_irk8u/tmpe6i67d4s')
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: pulp [3f22f7cd-bbfd-4eb2-99b3-2f5a36a89d38]: pulpcore.tasking.pulpcore_worker:INFO:   File "/usr/lib/python3.6/site-packages/pulpcore/tasking/pulpcore_worker.py", line 317, in _perform_task
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: result = func(*args, **kwargs)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulp_rpm/app/tasks/synchronizing.py", line 496, in synchronize
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: add_metadata_to_publication(publication, version)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulp_rpm/app/tasks/synchronizing.py", line 163, in add_metadata_to_publication
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: publication=publication,
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/app/models/publication.py", line 273, in create_from_file
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: artifact.save()
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/app/models/content.py", line 143, in save
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: super().save(*args, **kwargs)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django_lifecycle/mixins.py", line 134, in save
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: save(*args, **kwargs)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 744, in save
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: force_update=force_update, update_fields=update_fields)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 782, in save_base
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: force_update, using, update_fields,
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 851, in _save_table
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: for f in non_pks]
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 851, in <listcomp>
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: for f in non_pks]
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/app/models/fields.py", line 68, in pre_save
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: return super().pre_save(model_instance, add)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/fields/files.py", line 289, in pre_save
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: file.save(file.name, file.file, save=False)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/fields/files.py", line 88, in save
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: self.name = self.storage.save(name, content, max_length=self.field.max_length)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/core/files/storage.py", line 54, in save
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: return self._save(name, content)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/app/models/storage.py", line 69, in _save
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: file_move_safe(content.temporary_file_path(), full_path)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/core/files/move.py", line 56, in file_move_safe
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: with open(old_file_name, 'rb') as old_file:
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: Traceback (most recent call last):
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/bin/pulpcore-worker", line 11, in <module>
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: load_entry_point('pulpcore==3.14.7', 'console_scripts', 'pulpcore-worker')()
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/click/core.py", line 829, in __call__
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: return self.main(*args, **kwargs)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/click/core.py", line 782, in main
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: rv = self.invoke(ctx)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: return ctx.invoke(self.callback, **ctx.params)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/click/core.py", line 610, in invoke
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: return callback(*args, **kwargs)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/tasking/entrypoint.py", line 44, in worker
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: NewPulpWorker().run_forever()
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/tasking/pulpcore_worker.py", line 273, in run_forever
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: self.supervise_task(task)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/tasking/pulpcore_worker.py", line 223, in supervise_task
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: self.beat()
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/tasking/pulpcore_worker.py", line 100, in beat
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: self.worker_cleanup()
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/pulpcore/tasking/pulpcore_worker.py", line 90, in worker_cleanup
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: worker.delete()
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django_lifecycle/mixins.py", line 146, in delete
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: value = super().delete(*args, **kwargs)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/base.py", line 921, in delete
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: collector.collect([self], keep_parents=keep_parents)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/deletion.py", line 224, in collect
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: field.remote_field.on_delete(self, field, sub_objs, self.using)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/deletion.py", line 16, in CASCADE
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: source_attr=field.name, nullable=field.null)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/deletion.py", line 224, in collect
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: field.remote_field.on_delete(self, field, sub_objs, self.using)
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: File "/usr/lib/python3.6/site-packages/django/db/models/deletion.py", line 27, in PROTECT
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: sub_objs
Oct 28 14:03:01 katello.fqdn pulpcore-worker-1: django.db.models.deletion.ProtectedError: ("Cannot delete some instances of model 'ReservedResource' because they are referenced through a protected foreign key: 'TaskReservedResource.resource'", <QuerySet [<TaskReservedResource: pk=5b500325-8f56-42e7-b68e-4c66a16c8541>]>)

Ist this problem related ? Should I open a seperate Issue for that ?
Thanks a lot for you help

sasskinn12 · October 28, 2021, 12:20pm

Thanks a lot!

The patch indeed fixes this issue! (i had the same problem/error as other guys here).
For me everything works as it should right now. I can sync repos and publish packages in content view’s.

Justin_Sherrill · October 28, 2021, 12:23pm

Glad its working! I’ll work on getting those patches merged and get a 4.2.1 release out ASAP. @obsti that error looks unrelated. Lets open a 2nd ticket and we can dig deeper there.

obsti · October 29, 2021, 8:21am

Ok. I created a new Issue for my failure: