chore(ruff): cleanup indexer_* code after ruff analysis.

indexer_common.py:
  use 'for x, y in adict.items() rather than 'for x in adict' and
    later reference 'adict[x]'.

indexer_dbm.py:

  use 'with open() ... as fd: <stuff>' rather than 'fd =
    open()<stuff>close()'. 4 instances.

  whitespace fixes around operators

  use 'for x, y in adict.items() rather than 'for x in adict' and
    later reference 'adict[x]'.

  replace 'return 0' with 'return' as ruff was reporting function didn't
     have explicit return at end when internal return returned a not
     None value. AFICT load_index's return value is never checked.

  replace 'for _key, value in x.items()' with 'for value in
    x.values()'. _key is never used.


indexer_postgresql_fts.py:

  import sorting.

  symbolic string for server version needed for FTS support.

  rename variable id.

  remove unneeded flow control (else after raise)


indexer_rdbms.py:

  import sorting.

  rename variable id 2 places.

  add variable 'a = self.db.arg' to make scanning SQL code that uses %
    to insert placeholders easier. Also reduces dereferencing.

  replaced 'tuple([list comprehension])' with tuple(same comprehension
    but is a generator)

indexer_sqlite_fts.py:

  import sorting.

  rename variable id.

  remove unneeded flow control (else after raise).

indexer_whoosh.py:

  import sorting.

  remove qparser from whoosh import. Unused andnot needed until we
     implement using whoosh query language in roundup searches.

indexer_xapian.py:

  import sorting.

  rewrite loop that appended to a list as a list comprehension

Author: rouilj

CHANGES.txt

@@ -80,6 +80,7 @@ Fixed:
 - fix back_anydm::Class::get() method to properly return default value
   if requested property is set to None. This should fix missing text
   indexing for the anydbm backend. (John Rouillard)
+- ruff driven cleanups/refactor of indexer* files. (John Rouillard)
 
 Features:
 

roundup/backends/indexer_common.py

@@ -100,15 +100,15 @@ def search(self, search_terms, klass, ignore=None):
             nodeids[resid] = {}
             node_dict = nodeids[resid]
             # now figure out where it came from
-            for linkprop in propspec:
+            for linkprop, linkprop_value in propspec.items():
                 v = klass.get(resid, linkprop)
                 # the link might be a Link so deal with a single result or None
                 if isinstance(propdefs[linkprop], hyperdb.Link):
                     if v is None:
                         continue
                     v = [v]
                 for nodeid in v:
-                    if nodeid in propspec[linkprop]:
+                    if nodeid in linkprop_value:
                         # OK, this node[propname] has a winner
                         if linkprop not in node_dict:
                             node_dict[linkprop] = [nodeid]

roundup/backends/indexer_dbm.py

@@ -57,9 +57,8 @@ def __init__(self, db):
             # for now the file itself is a flag
             self.force_reindex()
         elif os.path.exists(version):
-            fd = open(version)
-            version = fd.read()
-            fd.close()
+            with open(version) as fd:
+                version = fd.read()
             # check the value and reindex if it's not the latest
             if version.strip() != '1':
                 self.force_reindex()
@@ -70,10 +69,9 @@ def force_reindex(self):
         if os.path.exists(self.indexdb_path):
             shutil.rmtree(self.indexdb_path)
         os.makedirs(self.indexdb_path)
-        os.chmod(self.indexdb_path, 0o775)  # nosec - allow group write
-        fd = open(os.path.join(self.indexdb_path, 'version'), 'w')
-        fd.write('1\n')
-        fd.close()
+        os.chmod(self.indexdb_path, 0o775)  # noqa: S103 allow group write
+        with open(os.path.join(self.indexdb_path, 'version'), 'w') as fd:
+            fd.write('1\n')
         self.reindex = 1
         self.changed = 1
 
@@ -98,7 +96,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
 
         # Find new file index, and assign it to identifier
         # (_TOP uses trick of negative to avoid conflict with file index)
-        self.files['_TOP'] = (self.files['_TOP'][0]-1, None)
+        self.files['_TOP'] = (self.files['_TOP'][0] - 1, None)
         file_index = abs(self.files['_TOP'][0])
         self.files[identifier] = (file_index, len(words))
         self.fileids[file_index] = identifier
@@ -109,12 +107,12 @@ def add_text(self, identifier, text, mime_type='text/plain'):
             if self.is_stopword(word):
                 continue
             if word in filedict:
-                filedict[word] = filedict[word]+1
+                filedict[word] = filedict[word] + 1
             else:
                 filedict[word] = 1
 
         # now add to the totals
-        for word in filedict:
+        for word, word_dict in filedict.items():
             # each word has a dict of {identifier: count}
             if word in self.words:
                 entry = self.words[word]
@@ -124,7 +122,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
                 self.words[word] = entry
 
             # make a reference to the file for this word
-            entry[file_index] = filedict[word]
+            entry[file_index] = word_dict
 
         # save needed
         self.changed = 1
@@ -165,7 +163,7 @@ def find(self, wordlist):
             if not self.minlength <= len(word) <= self.maxlength:
                 # word outside the bounds of what we index - ignore
                 continue
-            word = word.upper()
+            word = word.upper()  # noqa: PLW2901  # set loop var is ok
             if self.is_stopword(word):
                 continue
             entry = self.words.get(word)    # For each word, get index
@@ -192,7 +190,7 @@ def find(self, wordlist):
     def load_index(self, reload=0, wordlist=None):
         # Unless reload is indicated, do not load twice
         if self.index_loaded() and not reload:
-            return 0
+            return
 
         # Ok, now let's actually load it
         db = {'WORDS': {}, 'FILES': {'_TOP': (0, None)}, 'FILEIDS': {}}
@@ -211,13 +209,13 @@ def load_index(self, reload=0, wordlist=None):
         # Load the segments
         for segment in segments:
             try:
-                f = open(self.indexdb + segment, 'rb')
-            except IOError as error:
+                with open(self.indexdb + segment, 'rb') as f:
+                    pickle_str = zlib.decompress(f.read())
+            except IOError as error:  # noqa: PERF203 allow except inside loop
                 # probably just nonexistent segment index file
                 if error.errno != errno.ENOENT: raise          # noqa: E701
             else:
-                pickle_str = zlib.decompress(f.read())
-                f.close()
+                # FIXME 3.13 add allow_code=False to call
                 dbslice = marshal.loads(pickle_str)
                 if dbslice.get('WORDS'):
                     # if it has some words, add them
@@ -244,15 +242,14 @@ def save_index(self):
         for segment in self.segments:
             try:
                 os.remove(self.indexdb + segment)
-            except OSError as error:
+            except OSError as error:   # noqa: PERF203 allow except inside loop
                 # probably just nonexistent segment index file
                 if error.errno != errno.ENOENT: raise           # noqa: E701
 
         # First write the much simpler filename/fileid dictionaries
         dbfil = {'WORDS': None, 'FILES': self.files, 'FILEIDS': self.fileids}
-        marshal_fh = open(self.indexdb+'-', 'wb')
-        marshal_fh.write(zlib.compress(marshal.dumps(dbfil)))
-        marshal_fh.close()
+        with open(self.indexdb + '-', 'wb') as marshal_fh:
+            marshal_fh.write(zlib.compress(marshal.dumps(dbfil)))
 
         # The hard part is splitting the word dictionary up, of course
         letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_"
@@ -271,10 +268,9 @@ def save_index(self):
             db = {'WORDS': segdicts[initchar], 'FILES': None, 'FILEIDS': None}
             pickle_str = marshal.dumps(db)
             filename = self.indexdb + initchar
-            pickle_fh = open(filename, 'wb')
-            pickle_fh.write(zlib.compress(pickle_str))
-            pickle_fh.close()
-            os.chmod(filename, 0o664)
+            with open(filename, 'wb') as pickle_fh:
+                pickle_fh.write(zlib.compress(pickle_str))
+            os.chmod(filename, 0o664) # noqa: S103 allow group write
 
         # save done
         self.changed = 0
@@ -292,7 +288,7 @@ def purge_entry(self, identifier):
         del self.fileids[file_index]
 
         # The much harder part, cleanup the word index
-        for _key, occurs in self.words.items():
+        for occurs in self.words.values():
             if file_index in occurs:
                 del occurs[file_index]
 

roundup/backends/indexer_postgresql_fts.py

@@ -6,19 +6,19 @@
 
 import re
 
+from psycopg2.errors import InFailedSqlTransaction, SyntaxError, UndefinedObject  # noqa: A004
+
 from roundup.backends.indexer_common import Indexer as IndexerBase
-from roundup.i18n import _
 from roundup.cgi.exceptions import IndexerQueryError
-
-from psycopg2.errors import InFailedSqlTransaction, SyntaxError, \
-                            UndefinedObject
+from roundup.i18n import _
 
 
 class Indexer(IndexerBase):
     def __init__(self, db):
         IndexerBase.__init__(self, db)
         self.db = db
-        if db.conn.server_version < 110000:
+        min_fts_server_version = 110000
+        if db.conn.server_version < min_fts_server_version:
             db.sql("select version()")
             server_descr = db.cursor.fetchone()
             raise ValueError("Postgres native_fts indexing requires postgres "
@@ -79,11 +79,11 @@ def add_text(self, identifier, text, mime_type='text/plain'):
                 # we get a ValueError. For right now ignore it.
                 pass
         else:
-            id = r[0]
+            row_id = r[0]
             sql = 'update __fts set _tsv=to_tsvector(%s, %s) where ctid=%s' % \
                   (a, a, a)
             self.db.cursor.execute(sql, (self.db.config['INDEXER_LANGUAGE'],
-                                         text, id))
+                                         text, row_id))
 
     def find(self, wordlist):
         """look up all the words in the wordlist.
@@ -106,21 +106,21 @@ def find(self, wordlist):
             sql = ('select _class, _itemid, _prop from __fts '
                    'where _tsv @@ to_tsquery(%s, %s)' % (a, a))
 
-        else:
-            if re.search(r'[<>!&|()*]', " ".join(wordlist)):
-                # assume this is a ts query processed by websearch_to_tsquery.
-                # since it has operator characters in it.
-                raise IndexerQueryError(_('You have non-word/operator '
+        elif re.search(r'[<>!&|()*]', " ".join(wordlist)):
+            # assume this is a ts query processed by websearch_to_tsquery.
+            # since it has operator characters in it.
+            raise IndexerQueryError(_(
+                'You have non-word/operator '
                 'characters "<>!&|()*" in your query. Did you want to '
                 'do a tsquery search and forgot to start it with "ts:"?'))
-            else:
-                sql = 'select _class, _itemid, _prop from __fts '\
-                      'where _tsv @@ websearch_to_tsquery(%s, %s)' % (a, a)
+        else:
+            sql = 'select _class, _itemid, _prop from __fts '\
+                'where _tsv @@ websearch_to_tsquery(%s, %s)' % (a, a)
 
         try:
             # tests supply a multi element word list. Join them.
             self.db.cursor.execute(sql, (self.db.config['INDEXER_LANGUAGE'],
-                                         " ".join(wordlist),))
+                                         " ".join(wordlist)))
         except SyntaxError as e:
             # reset the cursor as it's invalid currently
             # reuse causes an InFailedSqlTransaction
@@ -145,7 +145,6 @@ def find(self, wordlist):
                 raise ValueError(_("Check tracker config.ini for a bad "
                                    "indexer_language setting. Error is: %s") %
                                  e)
-            else:
-                raise
+            raise
 
         return self.db.cursor.fetchall()

roundup/backends/indexer_rdbms.py

@@ -4,8 +4,8 @@
 """
 import re
 
+from roundup.anypy.strings import u2s, us2u
 from roundup.backends.indexer_common import Indexer as IndexerBase
-from roundup.anypy.strings import us2u, u2s
 
 
 class Indexer(IndexerBase):
@@ -52,15 +52,15 @@ def add_text(self, identifier, text, mime_type='text/plain'):
         r = self.db.cursor.fetchone()
         if not r:
             # not previously indexed
-            id = self.db.newid('__textids')
+            text_id = self.db.newid('__textids')
             sql = 'insert into __textids (_textid, _class, _itemid, _prop)'\
                 ' values (%s, %s, %s, %s)' % (a, a, a, a)
-            self.db.cursor.execute(sql, (id, ) + identifier)
+            self.db.cursor.execute(sql, (text_id, ) + identifier)
         else:
-            id = int(r[0])
+            text_id = int(r[0])
             # clear out any existing indexed values
             sql = 'delete from __words where _textid=%s' % a
-            self.db.cursor.execute(sql, (id, ))
+            self.db.cursor.execute(sql, (text_id, ))
 
         # ok, find all the unique words in the text
         text = us2u(text, "replace")
@@ -77,7 +77,7 @@ def add_text(self, identifier, text, mime_type='text/plain'):
 
         # for each word, add an entry in the db
         sql = 'insert into __words (_word, _textid) values (%s, %s)' % (a, a)
-        words = [(word, id) for word in words]
+        words = [(word, text_id) for word in words]
         self.db.cursor.executemany(sql, words)
 
     def find(self, wordlist):
@@ -88,6 +88,7 @@ def find(self, wordlist):
         if not wordlist:
             return []
 
+        a = self.db.arg  # placeholder for prepared statement
         cap_wl = [word.upper() for word in wordlist
                   if self.minlength <= len(word) <= self.maxlength]
         clean_wl = [word for word in cap_wl if not self.is_stopword(word)]
@@ -97,17 +98,16 @@ def find(self, wordlist):
 
         if self.db.implements_intersect:
             # simple AND search
-            sql = 'select distinct(_textid) from __words where _word=%s' % (
-                self.db.arg)
-            sql = '\nINTERSECT\n'.join([sql]*len(clean_wl))
+            sql = 'select distinct(_textid) from __words where _word=%s' % a
+            sql = '\nINTERSECT\n'.join([sql] * len(clean_wl))
             self.db.cursor.execute(sql, tuple(clean_wl))
             r = self.db.cursor.fetchall()
             if not r:
                 return []
-            a = ','.join([self.db.arg] * len(r))
+            a = ','.join([a] * len(r))
             sql = 'select _class, _itemid, _prop from __textids '\
                 'where _textid in (%s)' % a
-            self.db.cursor.execute(sql, tuple([int(row[0]) for row in r]))
+            self.db.cursor.execute(sql, tuple(int(row[0]) for row in r))
 
         else:
             # A more complex version for MySQL since it doesn't
@@ -126,17 +126,17 @@ def find(self, wordlist):
             match_list = []
             for n in range(len(clean_wl) - 1):
                 join_list.append(join_tmpl % (n + 2))
-                match_list.append(match_tmpl % (n + 2, self.db.arg))
+                match_list.append(match_tmpl % (n + 2, a))
 
-            sql = sql % (' '.join(join_list), self.db.arg,
+            sql = sql % (' '.join(join_list), a,
                          ' '.join(match_list))
             self.db.cursor.execute(sql, clean_wl)
 
             r = [x[0] for x in self.db.cursor.fetchall()]
             if not r:
                 return []
 
-            a = ','.join([self.db.arg] * len(r))
+            a = ','.join([a] * len(r))
             sql = 'select _class, _itemid, _prop from __textids '\
                 'where _textid in (%s)' % a
 

roundup/backends/indexer_sqlite_fts.py

@@ -19,8 +19,8 @@
 """
 
 from roundup.backends.indexer_common import Indexer as IndexerBase
-from roundup.i18n import _
 from roundup.cgi.exceptions import IndexerQueryError
+from roundup.i18n import _
 
 try:
     import sqlite3 as sqlite
@@ -84,10 +84,10 @@ def add_text(self, identifier, text, mime_type='text/plain'):
                 ' values (%s, %s, %s, %s)' % (a, a, a, a)
             self.db.cursor.execute(sql, identifier + (text,))
         else:
-            id = int(r[0])
+            text_id = int(r[0])
             sql = 'update __fts set _textblob=%s where rowid=%s' % \
                   (a, a)
-            self.db.cursor.execute(sql, (text, id))
+            self.db.cursor.execute(sql, (text, text_id))
 
     def find(self, wordlist):
         """look up all the words in the wordlist.
@@ -132,8 +132,8 @@ def find(self, wordlist):
                 raise IndexerQueryError(
                     _("Search failed. Try quoting any terms that "
                       "include a '-' and retry the search."))
-            else:
-                raise IndexerQueryError(e.args[0].replace("fts5:",
-                                                          "Query error:"))
+
+            raise IndexerQueryError(e.args[0].replace("fts5:",
+                                                      "Query error:"))
 
         return self.db.cursor.fetchall()

roundup/backends/indexer_whoosh.py

@@ -2,10 +2,10 @@
 '''
 import os
 
-from whoosh import fields, qparser, index, query, analysis
+from whoosh import analysis, fields, index, query
 
-from roundup.backends.indexer_common import Indexer as IndexerBase
 from roundup.anypy.strings import us2u
+from roundup.backends.indexer_common import Indexer as IndexerBase
 
 
 class Indexer(IndexerBase):