use pacfiles instead of alpmfiles if available

Huge performance improvements:

nvchecker.scope: Consumed 2.015s CPU time over 2.196s wall clock time, 942M memory peak, 113.4M read from disk.
nvchecker.scope: Consumed 342ms CPU time over 361ms wall clock time, 38.6M memory peak, 12.2M read from disk.

The above is just for one package. pacfiles avoids OOM when
running concurrently.

Author: lilydjwg

lilac2/lilacyaml.py

@@ -67,13 +67,15 @@ def load_lilac_yaml(dir: Path) -> dict[str, Any]:
 
   return conf
 
-def load_managed_lilacinfos(repodir: Path) -> tuple[LilacInfos, dict[str, ExcInfo]]:
+def load_managed_lilacinfos(
+  repodir: Path, use_pacfiles: bool = False,
+) -> tuple[LilacInfos, dict[str, ExcInfo]]:
   infos: LilacInfos = {}
   errors = {}
 
   for x in iter_pkgdir(repodir):
     try:
-      info = load_lilacinfo(x)
+      info = load_lilacinfo(x, use_pacfiles=use_pacfiles)
       if not info.managed:
         continue
       if info.time_limit_hours < 0:
@@ -84,10 +86,10 @@ def load_managed_lilacinfos(repodir: Path) -> tuple[LilacInfos, dict[str, ExcInf
 
   return infos, errors
 
-def load_lilacinfo(dir: Path) -> LilacInfo:
+def load_lilacinfo(dir: Path, use_pacfiles: bool = False) -> LilacInfo:
   yamlconf = load_lilac_yaml(dir)
   if update_on := yamlconf.get('update_on'):
-    update_ons, throttle_info = parse_update_on(update_on)
+    update_ons, throttle_info = parse_update_on(update_on, use_pacfiles=use_pacfiles)
   else:
     update_ons = []
     throttle_info = {}
@@ -114,6 +116,7 @@ def expand_alias_arg(value: str) -> str:
 
 def parse_update_on(
   update_on: list[dict[str, Any]],
+  use_pacfiles: bool = False,
 ) -> tuple[NvEntries, dict[int, datetime.timedelta]]:
   ret_update: NvEntries = []
   ret_throttle = {}
@@ -149,6 +152,9 @@ def parse_update_on(
     if source == 'alpm' or source == 'alpmfiles':
       entry.setdefault('dbpath', str(PACMAN_DB_DIR))
 
+    if use_pacfiles and source == 'alpmfiles':
+      entry['source'] = 'pacfiles'
+
     ret_update.append(entry)
 
   return ret_update, ret_throttle

lilac2/pkgbuild.py

@@ -17,6 +17,7 @@
 from .const import _G, OFFICIAL_REPOS
 from .cmd import UNTRUSTED_PREFIX
 from .typing import PkgVers
+from .tools import has_pacfiles
 
 logger = logging.getLogger(__name__)
 _official_packages: Dict[str, int] = {}
@@ -53,8 +54,11 @@ def _save_timed_dict(
   data_str = ''.join(f'{k} {v}\n' for k, v in data.items())
   safe_overwrite(str(path), data_str, mode='w')
 
-def update_pacmandb(dbpath: Path, pacman_conf: Optional[str] = None,
-                    *, quiet: bool = False) -> None:
+def update_pacmandb(
+  dbpath: Path, pacman_conf: Optional[str] = None, *,
+  quiet: bool = False,
+  update_pacfiles: bool = False,
+) -> None:
   stdout = subprocess.DEVNULL if quiet else None
 
   for update_arg in ['-Sy', '-Fy']:
@@ -73,11 +77,18 @@ def update_pacmandb(dbpath: Path, pacman_conf: Optional[str] = None,
     else:
       p.check_returncode()
 
-def update_data(pacman_conf: Optional[str],
-                *, quiet: bool = False) -> None:
+  if update_pacfiles:
+    cmd = ['pacfiles', '--dbpath', dbpath, '--update-db']
+    subprocess.check_call(cmd, stdout = stdout)
+
+def update_data(
+  pacman_conf: Optional[str], *,
+  quiet: bool = False, update_pacfiles: bool = False,
+) -> None:
   from .const import PACMAN_DB_DIR
   dbpath = PACMAN_DB_DIR
-  update_pacmandb(dbpath, pacman_conf, quiet=quiet)
+  update_pacmandb(dbpath, pacman_conf,
+                  quiet=quiet, update_pacfiles=update_pacfiles)
 
   now = int(time.time())
   deadline = now - 90 * 86400
@@ -177,4 +188,4 @@ def _get_package_version(srcinfo: List[str]) -> PkgVers:
 if __name__ == '__main__':
   import sys
   conf = sys.argv[1] if len(sys.argv) == 2 else None
-  update_data(conf)
+  update_data(conf, update_pacfiles=has_pacfiles())

lilac2/repo.py

@@ -19,7 +19,7 @@
 
 from .mail import MailService
 from .packages import get_built_package_files
-from .tools import ansi_escape_re
+from .tools import ansi_escape_re, has_pacfiles
 from . import lilacyaml, intl
 from .typing import LilacMod, Maintainer, LilacInfos, LilacInfo
 from .nomypy import BuildResult # type: ignore
@@ -352,7 +352,8 @@ def manages(self, dep: Dependency) -> bool:
     return dep.pkgdir.name in self.lilacinfos
 
   def load_managed_lilac_and_report(self) -> dict[str, tuple[str, ...]]:
-    self.lilacinfos, errors = lilacyaml.load_managed_lilacinfos(self.repodir)
+    self.lilacinfos, errors = lilacyaml.load_managed_lilacinfos(
+      self.repodir, use_pacfiles=has_pacfiles())
     failed: dict[str, tuple[str, ...]] = {p: () for p in errors}
     l10n = intl.get_l10n('mail')
     for name, exc_info in errors.items():

lilac2/tools.py

@@ -51,6 +51,15 @@ def get_avail_memory() -> int:
         return int(l.split()[1]) * 1024
   return 10 *  1024 ** 3
 
+_HAS_PACFILES = None
+
+def has_pacfiles() -> bool:
+  global _HAS_PACFILES
+  if _HAS_PACFILES is None:
+    cmd = subprocess.run(['pacfiles', '--help'], stdout=subprocess.DEVNULL)
+    _HAS_PACFILES = cmd.returncode == 0
+  return _HAS_PACFILES
+
 if __name__ == '__main__':
   cpu = get_running_task_cpu_ratio()
   mem = get_avail_memory()

lilac2/web.py

@@ -0,0 +1,38 @@
+from aiohttp import web
+
+async def get_current(request):
+  raise NotImplementedError
+
+async def watch_update(request):
+  raise NotImplementedError
+
+def setup_app(app):
+  app.router.add_get('/building/current', get_current)
+  app.router.add_get('/building/watch', watch_update)
+
+def main():
+  import argparse
+
+  from .vendor.nicelogger import enable_pretty_logging
+
+  parser = argparse.ArgumentParser(
+    description = 'HTTP services for build.archlinuxcn.org',
+  )
+  parser.add_argument('--port', default=9008, type=int,
+                      help='port to listen on')
+  parser.add_argument('--ip', default='127.0.0.1',
+                      help='address to listen on')
+  parser.add_argument('--loglevel', default='info',
+                      choices=['debug', 'info', 'warn', 'error'],
+                      help='log level')
+  args = parser.parse_args()
+
+  enable_pretty_logging(args.loglevel.upper())
+
+  app = web.Application()
+  setup_app(app)
+
+  web.run_app(app, host=args.ip, port=args.port)
+
+if __name__ == '__main__':
+  main()

lilac2/workerman.py

@@ -9,6 +9,7 @@
 
 from .typing import PkgToBuild, Rusages
 from .cmd import git_pull_override
+from .tools import has_pacfiles
 
 logger = logging.getLogger(__name__)
 
@@ -154,7 +155,7 @@ def prepare_batch(
   ) -> None:
     from . import pkgbuild
     logger.info('[%s] updating pacman databases', self.name)
-    pkgbuild.update_data(pacman_conf)
+    pkgbuild.update_data(pacman_conf, update_pacfiles=has_pacfiles())
 
   @override
   def finish_batch(self) -> None:

nvchecker_source/pacfiles.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2023 Pekka Ristola <pekkarr [at] protonmail [dot] com>, et al.
+
+from asyncio import create_subprocess_exec
+from asyncio.subprocess import PIPE
+import re
+from typing import Tuple, List
+
+from nvchecker.api import GetVersionError
+
+async def get_files(info: Tuple[str, str]) -> List[str]:
+  dbpath, pkg = info
+  # pacfiles is faster and memory-efficient
+  cmd = ['pacfiles', '-Flq', '--dbpath', dbpath, pkg]
+
+  p = await create_subprocess_exec(*cmd, stdout = PIPE, stderr = PIPE)
+  stdout, stderr = await p.communicate()
+
+  if p.returncode == 0:
+    return stdout.decode().splitlines()
+  else:
+    raise GetVersionError(
+      'pacfiles failed to get file list',
+      pkg = pkg,
+      cmd = cmd,
+      stdout = stdout.decode(errors='replace'),
+      stderr = stderr.decode(errors='replace'),
+      returncode = p.returncode,
+    )
+
+async def get_version(name, conf, *, cache, **kwargs):
+  pkg = conf['pkgname']
+  repo = conf.get('repo')
+  if repo is not None:
+    pkg = f'{repo}/{pkg}'
+  dbpath = conf.get('dbpath', '/var/lib/pacman')
+  regex = re.compile(conf['filename'])
+  if regex.groups > 1:
+    raise GetVersionError('multi-group regex')
+  strip_dir = conf.get('strip_dir', False)
+
+  files = await cache.get((dbpath, pkg), get_files)
+
+  for f in files:
+    fn = f.rsplit('/', 1)[-1] if strip_dir else f
+    match = regex.fullmatch(fn)
+    if match:
+      groups = match.groups()
+      return groups[0] if len(groups) > 0 else fn
+
+  raise GetVersionError('no file matches specified regex')
+
+