1.55.5 redo uuid system

Author: Enderbyte09

assets/control

@@ -1,5 +1,5 @@
 Package: craftserversetup
-Version: 1.55.4
+Version: 1.55.5
 Maintainer: Enderbyte Programs <enderbyte09@gmail.com>
 Homepage: https://github.com/Enderbyte-Programs/CraftServerSetup
 Architecture: all

changelog

@@ -1,3 +1,6 @@
+1.55.5:
+- Add UUID table export for first join and last seen
+- Overhaul UUID system
 1.55.4:
 - Add ability to launch a shell from manage server menu (Java and Bedrock)
 1.55.3:

export

@@ -14,14 +14,17 @@
 import logfilters
 import analyticsstructures
 import analyticsexplorer
-
-#TODO - split this file into analyticsexplorer.py and analyticsstructures.py
+import uuid
+from time import sleep
+import uuidmgr
 
 def analytics_loader(stdscr,serverdir):
     telemetry.telemetric_action("analytics")
     if uicomponents.resource_warning(stdscr):
         return
     renaminghandler.autoupdate_cache(stdscr,serverdir)
+    uuidmgr.load_from_logs(stdscr,serverdir)
+
     analytics_file_path = serverdir + os.sep + "anacache.json.gz"
     cursesplus.displaymsg(stdscr,["Loading from cache..."],False)
     readfrom = datetime.datetime(2000,1,1,0,0,0)
@@ -129,6 +132,7 @@ def analytics_loader(stdscr,serverdir):
 
     for k in final:
         final[k].onlineplayers = utils.remove_duplicates_from_list(final[k].onlineplayers)
+        final[k].onlineplayers = utils.recursive_remove_if_contains(final[k].onlineplayers,["\"","{","/"])
 
     #Find last zeroed time to store in cache
     storeto = analyticsstructures.get_minute_id_from_datetime(readfrom)#By default, store none. Then, try to find the last zero point
@@ -297,7 +301,7 @@ def analytics_loader(stdscr,serverdir):
         elif wtd == 10:
             plf = 0
             cursesplus.displaymsg(stdscr,["Analyzing data",f"{plf} players found"],False)
-            sortop = uicomponents.menu(stdscr,["Alphabetically","Recent -> Old"],"Choose search option")
+            sortop = uicomponents.menu(stdscr,["Alphabetically","Recent -> Old","Export UUIDs"],"Choose search option or data option")
             fjblock:dict[str,datetime.datetime] = {}
             for line in reversed(list(workingdata.values())):
                 for pl in line.onlineplayers:
@@ -306,6 +310,10 @@ def analytics_loader(stdscr,serverdir):
                         plf += 1
                         cursesplus.displaymsg(stdscr,["Analyzing data",f"{plf} players found"],False)
 
+            if sortop == 2:
+                do_date_based_uuid_export(stdscr,fjblock)
+                continue
+
             if sortop == 0:
                 fjblock = dict(sorted(fjblock.items()))#Sort A-Z
 
@@ -318,7 +326,7 @@ def analytics_loader(stdscr,serverdir):
         elif wtd == 11:
             plf = 0
             cursesplus.displaymsg(stdscr,["Analyzing data",f"{plf} players found"],False)
-            sortop = uicomponents.menu(stdscr,["Alphabetically","Oldest to newest"],"Choose search option")
+            sortop = uicomponents.menu(stdscr,["Alphabetically","Oldest to newest","Export UUIDs"],"Choose search option or data option")
             fjblock:dict[str,datetime.datetime] = {}
             for line in list(workingdata.values()):
                 for pl in line.onlineplayers:
@@ -327,6 +335,10 @@ def analytics_loader(stdscr,serverdir):
                         plf += 1
                         cursesplus.displaymsg(stdscr,["Analyzing data",f"{plf} players found"],False)
 
+            if sortop == 2:
+                do_date_based_uuid_export(stdscr,fjblock)
+                continue
+
             if sortop == 0:
                 fjblock = dict(sorted(fjblock.items()))#Sort A-Z
 
@@ -342,4 +354,43 @@ def analytics_loader(stdscr,serverdir):
             cursesplus.messagebox.showinfo(stdscr,["The analytics cache has been reset."])
             return
 
+def do_date_based_uuid_export(stdscr,vals:dict[str,datetime.datetime]) -> None:
+    """Produces and writes a file. Each line is one user. First block before space is uuid. All names are built to handle name changes"""
+    cursesplus.displaymsg(stdscr,["Please wait..."],False)
 
+    tempresult:dict[str,int] = {}
+
+    for kvpair in list(vals.items()):
+        latest_uname = renaminghandler.get_current_name_of(kvpair[0])
+        tempresult[latest_uname] = int(kvpair[1].timestamp())
+
+    cursesplus.displaymsg(stdscr,["It is now time to convert names into UUIDs","They will be processed at 1/second max","Press enter to proceed."])
+
+    result:dict[str,int] = {}
+    all_items = list(tempresult.items())
+    
+    pbar = cursesplus.ProgressBar(stdscr,len(all_items),bar_type=cursesplus.ProgressBarTypes.FullScreenProgressBar,show_log=True,message="Loading UUIDs")
+
+    while len(all_items) > 0:
+        
+        selitem = all_items.pop()
+        mresult = uuidmgr.get_uuid_from_name(selitem[0])
+        if mresult == "":
+            continue
+        
+        cursesplus.displaymsg(stdscr,[f"{len(all_items)} remaining"],False)
+        result[str(mresult)] = selitem[1]
+        pbar.step(f"{selitem[0]} -> {mresult}",True)
+
+    pbar.done()
+
+        
+    outfile = cursesplus.savefile_selector(stdscr)
+    with open(outfile,"w+") as f:
+        for resitem in list(result.items()):
+            f.write(resitem[0])
+            f.write(" ")
+            f.write(str(resitem[1]))
+            f.write("\n")
+    
+    cursesplus.messagebox.showinfo(stdscr,["Saved successfully"])

src/analytics.py

@@ -215,14 +215,10 @@ def compatibilize_appdata(data:dict) -> dict:
     "language" : None
 }
 
-UUID_INDEX:dict = {}
-
 def setup_appdata():
     """Load appdata from the file and populate the appdata.APPDATA variable."""
     global APPDATA
     global APPDATAFILE
-    global UUIDFILE
-    global UUID_INDEX
     APPDATAFILE = APPDATADIR+"/config.json"
     if not os.path.isfile(APPDATAFILE):
         with open(APPDATAFILE,"w+") as f:
@@ -238,19 +234,6 @@ def setup_appdata():
             APPDATA = __DEFAULTAPPDATA__
     APPDATA = compatibilize_appdata(APPDATA)
 
-    UUIDFILE = APPDATADIR+"/uuidindex.json.gz"
-
-    if not os.path.isfile(UUIDFILE):
-        with open(UUIDFILE,"wb+") as f:
-            f.write(gzip.compress(json.dumps({}).encode()))
-    else:
-        try:
-            with open(UUIDFILE,"rb") as f:
-                UUID_INDEX = json.loads(gzip.decompress(f.read()))
-        except:
-            with open(UUIDFILE,"wb+") as f:
-                f.write(gzip.compress(json.dumps({}).encode()))
-
 def self_compatibilize():
     """Execute a compatibilization and repair routine on appdata.APPDATA"""
     global APPDATA
@@ -261,12 +244,8 @@ def updateappdata():
     """Write in-memory appdata and UUID cache to the disk"""
 
     global APPDATAFILE
-    global UUID_INDEX
-    global UUIDFILE
     with open(APPDATAFILE,"w+") as f:
         f.write(json.dumps(APPDATA,indent=2))
-    with open(UUIDFILE,"wb+") as f:
-        f.write(gzip.compress(json.dumps(UUID_INDEX).encode()))#Write compressed file
 
 def get_setting_value(key:str) -> typing.Any:
     return APPDATA["settings"][key]["value"]

src/appdata.py

@@ -18,4 +18,7 @@ def is_logentry_a_command(l:logutils.LogEntry) -> bool:
     return len(re.findall(r"[a-zA-Z0-9_]+\sissued server command: .*",l.data)) > 0
 
 def player_rename(l:logutils.LogEntry) -> bool:
-    return len(re.findall(r"\S+ \(formerly known as \S+\)",l.data)) > 0
+    return len(re.findall(r"\S+ \(formerly known as \S+\)",l.data)) > 0
+
+def get_uuid(l:logutils.LogEntry) -> bool:
+    return len(re.findall(r": UUID of player \S+ is \S+$",l.data) + re.findall(r"Floodgate player logged in as \S+ joined \(UUID: \S+\)",l.data)) > 0

src/logfilters.py

@@ -104,24 +104,4 @@ def create_log_entry(data:str,fromfilename:str) -> logutils.LogEntry:
             int(fromfilename.split("-")[1]),
             int(fromfilename.split("-")[2])
         )
-    return logutils.LogEntry(fromfilename,ld,data)
-
-def load_server_logs_and_find(stdscr,serverdir:str,tofind:str) -> list[str]:
-    """Find specific matches of regex, removing the rest of the log entry"""
-    cursesplus.displaymsg(stdscr,["Loading Logs, Please wait..."],False)
-    logfile = serverdir + "/logs"
-    if not os.path.isdir(logfile):
-        return []
-    dirstack.pushd(logfile)
-    logs:list[str] = [l for l in os.listdir(logfile) if l.endswith(".gz") or l.endswith(".log")]
-    p = cursesplus.ProgressBar(stdscr,len(logs),cursesplus.ProgressBarTypes.SmallProgressBar,cursesplus.ProgressBarLocations.TOP,message="Loading logs")
-    final:list[str] = []
-    for lf in logs:
-        p.step(lf)
-        if lf.endswith(".gz"):
-            with open(lf,'rb') as f:
-                final.extend(re.findall(tofind,gzip.decompress(f.read()).decode()))
-        else:
-            with open(lf) as f:
-                final.extend(re.findall(tofind,f.read()))
-    return final
+    return logutils.LogEntry(fromfilename,ld,data)