From d07a47ff3c06f6e8b2adc21ae29eecae07badc9e Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Thu, 31 May 2018 01:27:45 -0700 Subject: [PATCH 1/4] emerge: make --nodeps disable --dynamic-deps (bug 656942) Since --nodeps disable's the depgraph's _dynamic_deps_preload code, calls to BlockerDB can trigger inappropriate calls to the FakeVartree _aux_get_wrapper method, triggering event loop recursion. Therefore, make --nodeps disable --dynamic-deps, in order to eliminate the inappropriate _aux_get_wrapper calls. Bug: https://bugs.gentoo.org/656942 --- pym/_emerge/create_depgraph_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/_emerge/create_depgraph_params.py b/pym/_emerge/create_depgraph_params.py index 1fd1f5e362..08605baa1f 100644 --- a/pym/_emerge/create_depgraph_params.py +++ b/pym/_emerge/create_depgraph_params.py @@ -48,7 +48,7 @@ def create_depgraph_params(myopts, myaction): myparams["ignore_soname_deps"] = myopts.get( "--ignore-soname-deps", "y") - dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" + dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" and "--nodeps" not in myopts if dynamic_deps: myparams["dynamic_deps"] = True From 345256c2d439c5ab580e4226f227db2819883d40 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Mon, 4 Jun 2018 16:32:46 -0700 Subject: [PATCH 2/4] emerge -pf: spawn pkg_nofetch asynchronously (bug 657360) For pretend mode, fix doebuild to skip the spawn_nofetch call that would trigger event loop recursion, and spawn pkg_nofetch asynchronously. Bug: https://bugs.gentoo.org/657360 --- pym/_emerge/EbuildBuild.py | 16 ++++++++++++++-- pym/_emerge/EbuildFetchonly.py | 5 ++++- pym/portage/package/ebuild/doebuild.py | 15 +++++++-------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index 00d4680f5a..d9f7f6da77 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -142,8 +142,20 @@ def _prefetch_exit(self, prefetcher): pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() - self.returncode = retval - self.wait() + if retval == os.EX_OK: + self._current_task = None + self.returncode = os.EX_OK + self._async_wait() + else: + # For pretend mode, the convention it to execute + # pkg_nofetch and return a successful exitcode. + self._start_task(SpawnNofetchWithoutBuilddir( + background=self.background, + portdb=self.pkg.root_config.trees[self._tree].dbapi, + ebuild_path=self._ebuild_path, + scheduler=self.scheduler, + settings=self.settings), + self._default_final_exit) return else: fetcher = EbuildFetcher( diff --git a/pym/_emerge/EbuildFetchonly.py b/pym/_emerge/EbuildFetchonly.py index f88ea96ef2..eec2ad208d 100644 --- a/pym/_emerge/EbuildFetchonly.py +++ b/pym/_emerge/EbuildFetchonly.py @@ -25,7 +25,10 @@ def execute(self): listonly=self.pretend, fetchonly=1, fetchall=self.fetch_all, mydbapi=portdb, tree="porttree") - if rval != os.EX_OK: + # For pretend mode, this error message is suppressed, + # and the unsuccessful return value is used to trigger + # a call to the pkg_nofetch phase. + if rval != os.EX_OK and not self.pretend: msg = "Fetch failed for '%s'" % (pkg.cpv,) eerror(msg, phase="unpack", key=pkg.cpv) diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index dc443df003..0e94de805b 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -1079,14 +1079,13 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, if not fetch(fetchme, mysettings, listonly=listonly, fetchonly=fetchonly, allow_missing_digests=False, digests=dist_digests): - spawn_nofetch(mydbapi, myebuild, settings=mysettings, - fd_pipes=fd_pipes) - if listonly: - # The convention for listonly mode is to report - # success in any case, even though fetch() may - # return unsuccessfully in order to trigger the - # nofetch phase. - return 0 + # Since listonly mode is called by emerge --pretend in an + # asynchronous context, spawn_nofetch would trigger event loop + # recursion here, therefore delegate execution of pkg_nofetch + # to the caller (bug 657360). + if not listonly: + spawn_nofetch(mydbapi, myebuild, settings=mysettings, + fd_pipes=fd_pipes) return 1 if need_distfiles: From e8067a8e6fbdaccca5915e66c77518e82b090401 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 5 Jun 2018 13:41:40 -0700 Subject: [PATCH 3/4] emerge --depclean: 'str' has no attribute 'soname' (bug 657420) Convert str to Atom, in order to avoid an AttributeError in the DbapiProvidesIndex.match method. Also, add comment explaining the reason for _unicode(atom) usage here, since it's not obvious. Bug: https://bugs.gentoo.org/657420 --- pym/_emerge/actions.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 70fb8d3b4d..f7232341d2 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -944,8 +944,23 @@ def unresolved_deps(): msg.append("the following required packages not being installed:") msg.append("") for atom, parent in unresolvable: + # For readability, we want to display the atom with USE + # conditionals evaluated whenever possible. However, + # there is a very special case where the atom does not + # match because the unevaluated form contains one or + # more flags for which the target package has missing + # IUSE, but due to conditionals those flags are only + # visible in the unevaluated form of the atom. In this + # case, we must display the unevaluated atom, so that + # the user can see the conditional USE deps that would + # otherwise be invisible. Use Atom(_unicode(atom)) to + # test for a package where this case would matter. This + # is not necessarily the same as atom.without_use, + # since Atom(_unicode(atom)) may still contain some + # USE dependencies that remain after evaluation of + # conditionals. if atom.package and atom != atom.unevaluated_atom and \ - vardb.match(_unicode(atom)): + vardb.match(Atom(_unicode(atom))): msg.append(" %s (%s) pulled in by:" % (atom.unevaluated_atom, atom)) else: From 937d0156aa060bdba9095313dedbb62e0a993aea Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Tue, 5 Jun 2018 18:52:59 -0700 Subject: [PATCH 4/4] CompositeTask: handle SIGINT/TERM cancelled futures (bug 657436) In order to avoid raising an unwanted CancelledError, make CompositeTask check for cancelled futures before attempting to call future.result(). The intention is only to handle failures triggered by SIGINT/TERM, since other types of expected failures should always be handled by catching the exception raised from future.result(). Bug: https://bugs.gentoo.org/657436 --- pym/_emerge/Binpkg.py | 10 +++++++++- pym/_emerge/BinpkgFetcher.py | 8 ++++++++ pym/_emerge/CompositeTask.py | 1 + pym/_emerge/EbuildBuild.py | 16 ++++++++++++++++ pym/_emerge/EbuildFetcher.py | 12 ++++++++++++ pym/_emerge/EbuildPhase.py | 4 ++++ pym/_emerge/PackageUninstall.py | 8 ++++++++ pym/portage/_emirrordist/FetchIterator.py | 10 ++++++++++ 8 files changed, 68 insertions(+), 1 deletion(-) diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py index 2b67816e81..7791ec2365 100644 --- a/pym/_emerge/Binpkg.py +++ b/pym/_emerge/Binpkg.py @@ -122,6 +122,10 @@ def _prefetch_exit(self, prefetcher): def _start_fetcher(self, lock_task=None): if lock_task is not None: self._assert_current(lock_task) + if lock_task.cancelled: + self._default_final_exit(lock_task) + return + lock_task.future.result() # Initialize PORTAGE_LOG_FILE (clean_log won't work without it). portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) @@ -411,8 +415,12 @@ def _async_unlock_builddir(self, returncode=None): def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) + if unlock_task.cancelled and returncode is not None: + self._default_final_exit(unlock_task) + return + # Normally, async_unlock should not raise an exception here. - unlock_task.future.result() + unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() diff --git a/pym/_emerge/BinpkgFetcher.py b/pym/_emerge/BinpkgFetcher.py index 8e651a1c7e..36d027de3c 100644 --- a/pym/_emerge/BinpkgFetcher.py +++ b/pym/_emerge/BinpkgFetcher.py @@ -48,6 +48,10 @@ def _start(self): def _start_locked(self, fetcher, lock_task): self._assert_current(lock_task) + if lock_task.cancelled: + self._default_final_exit(lock_task) + return + lock_task.future.result() self._start_task(fetcher, self._fetcher_exit) @@ -65,6 +69,10 @@ def _fetcher_exit(self, fetcher): def _fetcher_exit_unlocked(self, fetcher, unlock_task=None): if unlock_task is not None: self._assert_current(unlock_task) + if unlock_task.cancelled: + self._default_final_exit(unlock_task) + return + unlock_task.future.result() self._current_task = None diff --git a/pym/_emerge/CompositeTask.py b/pym/_emerge/CompositeTask.py index 4662f0cf53..1edec4a178 100644 --- a/pym/_emerge/CompositeTask.py +++ b/pym/_emerge/CompositeTask.py @@ -69,6 +69,7 @@ def _default_exit(self, task): self._assert_current(task) if task.returncode != os.EX_OK: self.returncode = task.returncode + self.cancelled = task.cancelled self._current_task = None return task.returncode diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index d9f7f6da77..8d264dd1cf 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -54,6 +54,10 @@ def _start(self): def _start_with_metadata(self, aux_get_task): self._assert_current(aux_get_task) + if aux_get_task.cancelled: + self._default_final_exit(aux_get_task) + return + pkg = self.pkg settings = self.settings root_config = pkg.root_config @@ -178,6 +182,10 @@ def _prefetch_exit(self, prefetcher): def _start_pre_clean(self, lock_task): self._assert_current(lock_task) + if lock_task.cancelled: + self._default_final_exit(lock_task) + return + lock_task.future.result() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. @@ -235,6 +243,10 @@ def _pre_clean_exit(self, pre_clean_phase): def _start_fetch(self, fetcher, already_fetched_task): self._assert_current(already_fetched_task) + if already_fetched_task.cancelled: + self._default_final_exit(already_fetched_task) + return + try: already_fetched = already_fetched_task.future.result() except portage.exception.InvalidDependString as e: @@ -342,6 +354,10 @@ def _async_unlock_builddir(self, returncode=None): def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) + if unlock_task.cancelled: + self._default_final_exit(unlock_task) + return + # Normally, async_unlock should not raise an exception here. unlock_task.future.result() if returncode is not None: diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py index 3b30ebb599..ad5109c285 100644 --- a/pym/_emerge/EbuildFetcher.py +++ b/pym/_emerge/EbuildFetcher.py @@ -47,6 +47,10 @@ def _start(self): def _start_fetch(self, uri_map_task): self._assert_current(uri_map_task) + if uri_map_task.cancelled: + self._default_final_exit(uri_map_task) + return + try: uri_map = uri_map_task.future.result() except portage.exception.InvalidDependString as e: @@ -71,6 +75,10 @@ def _start_fetch(self, uri_map_task): def _start_with_metadata(self, aux_get_task): self._assert_current(aux_get_task) + if aux_get_task.cancelled: + self._default_final_exit(aux_get_task) + return + self._fetcher_proc.src_uri, = aux_get_task.future.result() self._start_task(self._fetcher_proc, self._default_final_exit) @@ -85,6 +93,10 @@ def async_already_fetched(self, settings): result = self.scheduler.create_future() def uri_map_done(uri_map_future): + if uri_map_future.cancelled(): + result.cancel() + return + if uri_map_future.exception() is not None or result.cancelled(): if not result.cancelled(): result.set_exception(uri_map_future.exception()) diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py index d057dc45e3..4104cefa70 100644 --- a/pym/_emerge/EbuildPhase.py +++ b/pym/_emerge/EbuildPhase.py @@ -211,6 +211,10 @@ def _ebuild_exit(self, ebuild_process): def _ebuild_exit_unlocked(self, ebuild_process, unlock_task=None): if unlock_task is not None: self._assert_current(unlock_task) + if unlock_task.cancelled: + self._default_final_exit(unlock_task) + return + # Normally, async_unlock should not raise an exception here. unlock_task.future.result() diff --git a/pym/_emerge/PackageUninstall.py b/pym/_emerge/PackageUninstall.py index 3fe1fb0a6f..cb34130562 100644 --- a/pym/_emerge/PackageUninstall.py +++ b/pym/_emerge/PackageUninstall.py @@ -61,6 +61,10 @@ def _start(self): def _start_unmerge(self, lock_task): self._assert_current(lock_task) + if lock_task.cancelled: + self._default_final_exit(lock_task) + return + lock_task.future.result() portage.prepare_build_dirs( settings=self.settings, cleanup=True) @@ -112,6 +116,10 @@ def _async_unlock_builddir(self, returncode=None): def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) + if unlock_task.cancelled: + self._default_final_exit(unlock_task) + return + # Normally, async_unlock should not raise an exception here. unlock_task.future.result() if returncode is not None: diff --git a/pym/portage/_emirrordist/FetchIterator.py b/pym/portage/_emirrordist/FetchIterator.py index 04d4da62b2..4ad7975029 100644 --- a/pym/portage/_emirrordist/FetchIterator.py +++ b/pym/portage/_emirrordist/FetchIterator.py @@ -135,11 +135,21 @@ def aux_get_done(gather_result): if not gather_result.cancelled(): list(future.exception() for future in gather_result.result() if not future.cancelled()) + else: + result.cancel() if result.cancelled(): return aux_get_result, fetch_map_result = gather_result.result() + if aux_get_result.cancelled() or fetch_map_result.cancelled(): + # Cancel result after consuming any exceptions which + # are now irrelevant due to cancellation. + aux_get_result.cancelled() or aux_get_result.exception() + fetch_map_result.cancelled() or fetch_map_result.exception() + result.cancel() + return + try: restrict, = aux_get_result.result() except (PortageKeyError, PortageException) as e: