cleanup tld db

Author: mboot-github

analizer/Makefile

@@ -35,16 +35,21 @@ COMMON_VENV := rm -rf $(VENV); \
 
 all: clean prep test
 
+# ---------------------------
 clean: cleanupVenv
 	rm -f *.1 *.2 *.log *.tmp  1 2
 	rm -rf .mypy_cache
 	# rm -f IanaDb.sqlite # dont remove during testing
 	rm -rf .iana_cache/ .psl_cache/
 
+realclean: clean
+	rm -f *.sqlite
+
 # cleanup the virtual env
 cleanupVenv:
 	rm -rf $(VENV)
 
+# ---------------------------
 prep: format check
 
 format:
@@ -53,10 +58,12 @@ format:
 check:
 	ruff check --fix $(PY_FILES)
 
+# ---------------------------
 test: IanaDb.sqlite
 	$(COMMON_VENV) \
 	$(PIP_INSTALL) -r requirements.txt ;\
 		./investigateTld.py 2>2 | tee 1
+		cat 2
 
 IanaDb.sqlite:
 	$(COMMON_VENV) \

analizer/investigateTld.py

@@ -21,13 +21,14 @@ def extract_server_hints(aDict: dict[str, Any]) -> dict[str, Any]:
     # key will be the whois server, data will be the list of tld's using this server
     servers: dict[str, Any] = {}
     k = "_server"
+
     for key, value in aDict.items():
         if k in value:
             server = value[k]
             if server not in servers:
                 servers[server] = []
             servers[server].append(key)
-    # print(servers)
+
     return servers
 
 
@@ -53,8 +54,6 @@ def xMain() -> None:
     dbFileName = "IanaDb.sqlite"
     allTld: list[str] = []
 
-    # print(tld_regexpr.ZZ)
-    # sys.exit(0)
     server_hints = extract_server_hints(tld_regexpr.ZZ)
 
     iad = IanaDatabase(verbose=verbose)
@@ -68,9 +67,10 @@ def xMain() -> None:
         tld = ot.tld
         forest[tld] = ot
 
-    for tld, data in forest.items():
-        if data.rdap_info != "NULL":
-            print(tld, data.rdap_info)
+    if 0:
+        for tld, data in forest.items():
+            if data.rdap_info != "NULL":
+                print("RDAP", tld, data.rdap_info)
 
     allTld = sorted(allTld)
     for tld in tld_regexpr.ZZ:

analizer/oneTld.py

@@ -155,14 +155,18 @@ def _doUtf8Preparations(self):
         return False
 
     def _skipKnowTld(self):
+        z = self.allKnownTldDict.get(self.tld, {}).get("_privateRegistry")
+        if z is not None:
+            return False
+
         if self.tld2 == self.tld and self.tld in self.allKnownTldDict:
             return True
 
         return self.tld2 in self.allKnownTldDict and self.tld in self.allKnownTldDict
 
     def _doNoManagerTld(self):
         if self.manager == "NULL":
-            print(f"no manager fir tld: {self.tld}")
+            print(f"no manager for tld: {self.tld}")
             if self.tld not in self.allKnownTldDict:
                 print(f'# ZZ["{self.tld}"] = ' + '{"_privateRegistry": True} # no manager')
 
@@ -254,6 +258,12 @@ def processRow(
             if n():
                 return
 
+        if "whois.identitydigital.services" in self.resolved_whois_servers:
+            return
+
+        if "tucowsregistry.net" in self.resolved_whois_servers:
+            return
+
         print(
             "# MISSING",
             self.tld,

analizer/requirements.txt

@@ -1,3 +1,5 @@
 dnspython
 beautifulsoup4
 requests-cache
+tld
+whodap

pyproject.toml

@@ -177,7 +177,7 @@
         "PLW0603", # Using the global statement to update `xxx` is discouraged
         "PLW0602", # Using global for `xxx` but no assignment is done
 
-        "RUF001", # String contains ambiguous  urf8 char
+        "RUF001", # String contains ambiguous utf8 char
         "RUF067", # `__init__` module should only contain docstrings and re-exports
 
         "S101", # Use of `assert` detected