Merge pull request #3894 from lonvia/country-names-with-word-lookup

Add normalized form of country names to coutry tokens in word table

Author: lonvia

packaging/nominatim-db/pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
-    "psycopg",
+    "psycopg<3.3",
     "python-dotenv",
     "jinja2",
     "pyYAML>=5.1",

src/nominatim_api/search/db_search_builder.py

@@ -374,7 +374,7 @@ def get_search_data(self, assignment: TokenAssignment) -> Optional[dbf.SearchDat
             tokens = self.get_country_tokens(assignment.country)
             if not tokens:
                 return None
-            sdata.set_strings('countries', tokens)
+            sdata.set_countries(tokens)
             sdata.penalty += self.query.get_in_word_penalty(assignment.country)
         elif self.details.countries:
             sdata.countries = dbf.WeightedStrings(self.details.countries,

src/nominatim_api/search/db_search_fields.py

@@ -244,6 +244,21 @@ def set_strings(self, field: str, tokens: List[Token]) -> None:
 
             setattr(self, field, wstrs)
 
+    def set_countries(self, tokens: List[Token]) -> None:
+        """ Set the WeightedStrings properties for countries. Multiple
+            entries for the same country are deduplicated and the minimum
+            penalty is used. Adapts the global penalty, so that the
+            minimum penalty is 0.
+        """
+        if tokens:
+            min_penalty = min(t.penalty for t in tokens)
+            self.penalty += min_penalty
+            countries: dict[str, float] = {}
+            for t in tokens:
+                cc = t.get_country()
+                countries[cc] = min(t.penalty - min_penalty, countries.get(cc, 10000))
+            self.countries = WeightedStrings(list(countries.keys()), list(countries.values()))
+
     def set_qualifiers(self, tokens: List[Token]) -> None:
         """ Set the qulaifier field from the given tokens.
         """

src/nominatim_api/search/icu_tokenizer.py

@@ -59,12 +59,16 @@ def get_category(self) -> Tuple[str, str]:
         assert self.info
         return self.info.get('class', ''), self.info.get('type', '')
 
-    def rematch(self, norm: str) -> None:
+    def get_country(self) -> str:
+        assert self.info
+        return cast(str, self.info.get('cc', ''))
+
+    def match_penalty(self, norm: str) -> float:
         """ Check how well the token matches the given normalized string
             and add a penalty, if necessary.
         """
         if not self.lookup_word:
-            return
+            return 0.0
 
         seq = difflib.SequenceMatcher(a=self.lookup_word, b=norm)
         distance = 0
@@ -75,7 +79,7 @@ def rematch(self, norm: str) -> None:
                 distance += max((ato-afrom), (bto-bfrom))
             elif tag != 'equal':
                 distance += abs((ato-afrom) - (bto-bfrom))
-        self.penalty += (distance/len(self.lookup_word))
+        return (distance/len(self.lookup_word))
 
     @staticmethod
     def from_db_row(row: SaRow) -> 'ICUToken':
@@ -330,9 +334,10 @@ def rerank_tokens(self, query: qmod.QueryStruct) -> None:
             norm = ''.join(f"{n.term_normalized}{'' if n.btype == qmod.BREAK_TOKEN else ' '}"
                            for n in query.nodes[start + 1:end + 1]).strip()
             for ttype, tokens in tlist.items():
-                if ttype != qmod.TOKEN_COUNTRY:
-                    for token in tokens:
-                        cast(ICUToken, token).rematch(norm)
+                for token in tokens:
+                    itok = cast(ICUToken, token)
+                    itok.penalty += itok.match_penalty(norm) * \
+                        (1 if ttype in (qmod.TOKEN_WORD, qmod.TOKEN_PARTIAL) else 2)
 
     def compute_break_penalties(self, query: qmod.QueryStruct) -> None:
         """ Set the break penalties for the nodes in the query.

src/nominatim_api/search/query.py

@@ -127,6 +127,12 @@ def get_category(self) -> Tuple[str, str]:
             category objects.
         """
 
+    @abstractmethod
+    def get_country(self) -> str:
+        """ Return the country code this tojen is associated with
+            (currently for country tokens only).
+        """
+
 
 @dataclasses.dataclass
 class TokenRange:

src/nominatim_db/tokenizer/icu_tokenizer.py

@@ -475,31 +475,34 @@ def _add_country_full_names(self, country_code: str, names: Sequence[PlaceName],
         assert self.conn is not None
         word_tokens = set()
         for name in names:
-            norm_name = self._search_normalized(name.name)
-            if norm_name:
-                word_tokens.add(norm_name)
+            norm_name = self._normalized(name.name)
+            token_name = self._search_normalized(name.name)
+            if norm_name and token_name:
+                word_tokens.add((token_name, norm_name))
 
         with self.conn.cursor() as cur:
             # Get existing names
-            cur.execute("""SELECT word_token, coalesce(info ? 'internal', false) as is_internal
+            cur.execute("""SELECT word_token,
+                                  word as lookup,
+                                  coalesce(info ? 'internal', false) as is_internal
                              FROM word
-                             WHERE type = 'C' and word = %s""",
+                             WHERE type = 'C' and info->>'cc' = %s""",
                         (country_code, ))
             # internal/external names
-            existing_tokens: Dict[bool, Set[str]] = {True: set(), False: set()}
+            existing_tokens: Dict[bool, Set[Tuple[str, str]]] = {True: set(), False: set()}
             for word in cur:
-                existing_tokens[word[1]].add(word[0])
+                existing_tokens[word[2]].add((word[0], word[1]))
 
             # Delete names that no longer exist.
             gone_tokens = existing_tokens[internal] - word_tokens
             if internal:
                 gone_tokens.update(existing_tokens[False] & word_tokens)
             if gone_tokens:
                 cur.execute("""DELETE FROM word
-                               USING unnest(%s::text[]) as token
-                               WHERE type = 'C' and word = %s
-                                     and word_token = token""",
-                            (list(gone_tokens), country_code))
+                               USING jsonb_array_elements(%s) as data
+                               WHERE type = 'C' and info->>'cc' = %s
+                                     and word_token = data->>0 and word = data->>1""",
+                            (Jsonb(list(gone_tokens)), country_code))
 
             # Only add those names that are not yet in the list.
             new_tokens = word_tokens - existing_tokens[True]
@@ -508,15 +511,17 @@ def _add_country_full_names(self, country_code: str, names: Sequence[PlaceName],
             if new_tokens:
                 if internal:
                     sql = """INSERT INTO word (word_token, type, word, info)
-                               (SELECT token, 'C', %s, '{"internal": "yes"}'
-                                  FROM unnest(%s::text[]) as token)
+                               (SELECT data->>0, 'C', data->>1,
+                                       jsonb_build_object('internal', 'yes', 'cc', %s::text)
+                                  FROM jsonb_array_elements(%s) as data)
                            """
                 else:
-                    sql = """INSERT INTO word (word_token, type, word)
-                                   (SELECT token, 'C', %s
-                                    FROM unnest(%s::text[]) as token)
+                    sql = """INSERT INTO word (word_token, type, word, info)
+                                   (SELECT data->>0, 'C', data->>1,
+                                           jsonb_build_object('cc', %s::text)
+                                    FROM  jsonb_array_elements(%s) as data)
                           """
-                cur.execute(sql, (country_code, list(new_tokens)))
+                cur.execute(sql, (country_code, Jsonb(list(new_tokens))))
 
     def process_place(self, place: PlaceInfo) -> Mapping[str, Any]:
         """ Determine tokenizer information about the given place.

src/nominatim_db/tools/migration.py

@@ -2,7 +2,7 @@
 #
 # This file is part of Nominatim. (https://nominatim.org)
 #
-# Copyright (C) 2024 by the Nominatim developer community.
+# Copyright (C) 2025 by the Nominatim developer community.
 # For a full list of authors see the git log.
 """
 Functions for database migration to newer software versions.
@@ -18,6 +18,7 @@
 from ..db.sql_preprocessor import SQLPreprocessor
 from ..version import NominatimVersion, NOMINATIM_VERSION, parse_version
 from ..tokenizer import factory as tokenizer_factory
+from ..data.country_info import create_country_names, setup_country_config
 from . import refresh
 
 LOG = logging.getLogger()
@@ -156,3 +157,25 @@ def create_place_entrance_table(conn: Connection, config: Configuration, **_: An
             CREATE UNIQUE INDEX place_entrance_osm_id_idx ON place_entrance
               USING BTREE (osm_id);
               """)
+
+
+@_migration(5, 2, 99, 1)
+def convert_country_tokens(conn: Connection, config: Configuration, **_: Any) -> None:
+    """ Convert country word tokens
+
+        Country tokens now save the country in the info field instead of the
+        word. This migration removes all country tokens from the word table
+        and reimports the default country name. This means that custom names
+        are lost. If you need them back, invalidate the OSM objects containing
+        the names by setting indexed_status to 2 and then reindex the database.
+    """
+    tokenizer = tokenizer_factory.get_tokenizer_for_db(config)
+    # There is only one tokenizer at the time of migration, so we make
+    # some assumptions here about the structure of the database. This will
+    # fail if somebody has written a custom tokenizer.
+    with conn.cursor() as cur:
+        cur.execute("DELETE FROM word WHERE type = 'C'")
+    conn.commit()
+
+    setup_country_config(config)
+    create_country_names(conn, tokenizer, config.get_str_list('LANGUAGES'))

src/nominatim_db/version.py

@@ -55,7 +55,7 @@ def parse_version(version: str) -> NominatimVersion:
     return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')])
 
 
-NOMINATIM_VERSION = parse_version('5.2.0-0')
+NOMINATIM_VERSION = parse_version('5.2.99-0')
 
 POSTGRESQL_REQUIRED_VERSION = (12, 0)
 POSTGIS_REQUIRED_VERSION = (3, 0)

test/python/api/search/test_api_search_query.py

@@ -17,6 +17,9 @@ class MyToken(query.Token):
     def get_category(self):
         return 'this', 'that'
 
+    def get_country(self):
+        return 'cc'
+
 
 def mktoken(tid: int):
     return MyToken(penalty=3.0, token=tid, count=1, addr_count=1,

test/python/api/search/test_db_search_builder.py

@@ -21,6 +21,9 @@ class MyToken(Token):
     def get_category(self):
         return 'this', 'that'
 
+    def get_country(self):
+        return self.lookup_word
+
 
 def make_query(*args):
     q = QueryStruct([Phrase(qmod.PHRASE_ANY, '')])