Repo sync fails, too many open files

Problem: I’m on Foreman 3.5 with Katello and I’m trying to sync the Rocky Linux 8 repos AppStream and BaseOS and getting [Errno 24] Too many open files: ‘./tmpjbq5ysqk’. I’ve tried this on 3 different Foreman servers and got the same result on every single one.

Expected outcome: Repos should sync

Foreman and Proxy versions: 3.5.1

Distribution and version: CentOS 8 Stream

Other relevant data:

Blockquote
Actions::Pulp3::Repository::Sync
Input:
{“repo_id”=>48,
“smart_proxy_id”=>1,
“options”=>{},
“remote_user”=>“admin”,
“remote_cp_user”=>“admin”,
“current_request_id”=>“fff2b476-cbe4-43b1-b452-2ab0d42057f1”,
“current_timezone”=>“America/New_York”,
“current_organization_id”=>1,
“current_location_id”=>2,
“current_user_id”=>4}

Blockquote
Output:
{“pulp_tasks”=>
[{“pulp_href”=>"/pulp/api/v3/tasks/3ebbfeef-b065-4946-b58d-d1b2194dcbd7/",
“pulp_created”=>“2023-02-09T15:45:38.284+00:00”,
“state”=>“failed”,
“name”=>“pulp_rpm.app.tasks.synchronizing.synchronize”,
“logging_cid”=>“fff2b476-cbe4-43b1-b452-2ab0d42057f1”,
“started_at”=>“2023-02-09T16:00:46.630+00:00”,
“finished_at”=>“2023-02-09T16:04:56.820+00:00”,
“error”=>
{“traceback”=>
#<Sequel::SQL::Blob:0x2e810 bytes=106 start=" File “/u” end=“form_task\n”> +
#<Sequel::SQL::Blob:0x2e824 bytes=35 start=" result" end="**kwargs)\n"> +
#<Sequel::SQL::Blob:0x2e838 bytes=104 start=" File “/u” end=“nchronize\n”> +
#<Sequel::SQL::Blob:0x2e84c bytes=56 start=" repo_v" end=“version()\n”> +
#<Sequel::SQL::Blob:0x2e860 bytes=109 start=" File “/u” end=“in create\n”> +
#<Sequel::SQL::Blob:0x2e874 bytes=38 start=" loop.r" end=“pipeline)\n”> +
#<Sequel::SQL::Blob:0x2e888 bytes=86 start=" File “/u” end="_complete\n"> +
#<Sequel::SQL::Blob:0x2e89c bytes=27 start=" return" end=".result()\n"> +
#<Sequel::SQL::Blob:0x2e8b0 bytes=102 start=" File “/u” end="_pipeline\n"> +
#<Sequel::SQL::Blob:0x2e8c4 bytes=35 start=" await " end="*futures)\n"> +
#<Sequel::SQL::Blob:0x2e8d8 bytes=94 start=" File “/u” end=" call\n"> +
#<Sequel::SQL::Blob:0x2e8ec bytes=21 start=" await " end=“elf.run()\n”> +
#<Sequel::SQL::Blob:0x2e900 bytes=102 start=" File “/u” end=“3, in run\n”> +
#<Sequel::SQL::Blob:0x2e914 bytes=47 start=" pb.don" end=“oad_count\n”> +
#<Sequel::SQL::Blob:0x2e928 bytes=119 start=" File “/u” end=“tent_unit\n”> +
#<Sequel::SQL::Blob:0x2e93c bytes=51 start=" await " end="_content)\n"> +
#<Sequel::SQL::Blob:0x2e950 bytes=98 start=" File “/u” end=" download\n"> +
#<Sequel::SQL::Blob:0x2e964 bytes=12 content=" raise e\n"> +
#<Sequel::SQL::Blob:0x2e978 bytes=98 start=" File “/u” end=" download\n"> +
#<Sequel::SQL::Blob:0x2e98c bytes=71 start=" downlo" end=“tra_data)\n”> +
#<Sequel::SQL::Blob:0x2e9a0 bytes=86 start=" File “/u” end=“3, in run\n”> +
#<Sequel::SQL::Blob:0x2e9b4 bytes=36 start=" return" end=“wrapper()\n”> +
#<Sequel::SQL::Blob:0x2e9c8 bytes=80 start=" File “/u” end=" in retry\n"> +
#<Sequel::SQL::Blob:0x2e9dc bytes=40 start=" ret = " end="**kwargs)\n"> +
#<Sequel::SQL::Blob:0x2e9f0 bytes=99 start=" File “/u” end=“d_wrapper\n”> +
#<Sequel::SQL::Blob:0x2ea04 bytes=50 start=" return" end=“tra_data)\n”> +
#<Sequel::SQL::Blob:0x2ea18 bytes=89 start=" File “/u” end=", in run\n"> +
#<Sequel::SQL::Blob:0x2ea2c bytes=54 start=" to_ret" end=“response)\n”> +
#<Sequel::SQL::Blob:0x2ea40 bytes=99 start=" File “/u” end=“response\n"> +
#<Sequel::SQL::Blob:0x2ea54 bytes=34 start=" await " end=“ta(chunk)\n”> +
#<Sequel::SQL::Blob:0x2ea68 bytes=94 start=" File “/u” end=“ndle_data\n”> +
#<Sequel::SQL::Blob:0x2ea7c bytes=40 start=" self.
” end=“en_file()\n”> +
#<Sequel::SQL::Blob:0x2ea90 bytes=111 start=" File “/u” end=“open_file\n”> +
#<Sequel::SQL::Blob:0x2eaa4 bytes=70 start=" self.
" end=“te=False)\n”> +
#<Sequel::SQL::Blob:0x2eab8 bytes=75 start=" File “/u” end=“oraryFile\n”> +
#<Sequel::SQL::Blob:0x2eacc bytes=73 start=" (fd, n" end=“put_type)\n”> +
#<Sequel::SQL::Blob:0x2eae0 bytes=71 start=" File “/u” end=“emp_inner\n”> +
#<Sequel::SQL::Blob:0x2eaf4 bytes=38 start=" fd = _" end=“s, 0o600)\n”>,
“description”=>"[Errno 24] Too many open files: ‘./tmpjbq5ysqk’"},
“worker”=>"/pulp/api/v3/workers/dad4e05d-579f-48c4-b487-e184ac3d25fe/",
“child_tasks”=>,
“progress_reports”=>
[{“message”=>“Downloading Metadata Files”,
“code”=>“sync.downloading.metadata”,
“state”=>“completed”,
“done”=>7},
{“message”=>“Associating Content”,
“code”=>“associating.content”,
“state”=>“canceled”,
“done”=>1000},
{“message”=>“Parsed Modulemd”,
“code”=>“sync.parsing.modulemds”,
“state”=>“completed”,
“total”=>125,
“done”=>125},
{“message”=>“Parsed Modulemd-defaults”,
“code”=>“sync.parsing.modulemd_defaults”,
“state”=>“completed”,
“total”=>47,
“done”=>47},
{“message”=>“Parsed Modulemd Obsolete”,
“code”=>“sync.parsing.modulemd_obsoletes”,
“state”=>“completed”,
“total”=>47,
“done”=>47},
{“message”=>“Skipping Packages”,
“code”=>“sync.skipped.packages”,
“state”=>“completed”,
“total”=>0,
“done”=>0},
{“message”=>“Downloading Artifacts”,
“code”=>“sync.downloading.artifacts”,
“state”=>“failed”,
“done”=>1341},
{“message”=>“Parsed Packages”,
“code”=>“sync.parsing.packages”,
“state”=>“canceled”,
“total”=>7345,
“done”=>2455}],
“created_resources”=>,
“reserved_resources_record”=>
["/pulp/api/v3/repositories/rpm/rpm/3b3c02ef-a271-4d97-81a6-8cc553f08565/",
“shared:/pulp/api/v3/remotes/rpm/rpm/02407058-64b6-4e7b-bf60-fb7f99c142f9/”]}],
“create_version”=>true,
“task_groups”=>,
“poll_attempts”=>{“total”=>92, “failed”=>1}}

We ended up using a different Repo from the mirror-list and it synced fine.