diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml index 8f5798c3..bf764cf4 100644 --- a/.github/workflows/build_tools.yml +++ b/.github/workflows/build_tools.yml @@ -42,13 +42,9 @@ jobs: - build_usermgmt steps: - name: packaging - run: | - echo "packaging the app now" - cd usermgmt\windows\dist - copy ../camera.ico . - copy ../stationmaint.ini.sample . - copy ../../README.md . - compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip + uses: Minionguyjpro/Inno-Setup-Action@v1.2.2 + with: + path: usermgmt\windows\cam_management_setup.iss release_usermgmt: name: release usermanagement runs-on: win10 @@ -60,23 +56,23 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.MM_PAT }} with: - file: c:\temp\ukmon_usermgmt.zip + file: e:\temp\setup_cameraMgmt.exe tags: true draft: true overwrite: true - tag_name: 2025.10.1 + tag_name: 2025.11.1 default_release_name: User Management Tool default_release_body_path: usermgmt\windows\release_notes.md cleanup_usermgmt: - name: clean up temp usermanagement files + name: clean up temp cameramgmt files runs-on: win10 continue-on-error: true needs: - release_usermgmt steps: - - name: delete temp usermgmt package + - name: delete temp cameramgmt package run: | - del c:\temp\ukmon_usermgmt.zip + del e:\temp\setup_cameraMgmt.exe build_fbtool: name: build fbcollector runs-on: win10 @@ -88,6 +84,7 @@ jobs: conda activate fbcollector cd fbCollector pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico + markdown2 ./README.md > README.html package_fbtool: name: package fbcollector runs-on: win10 @@ -95,15 +92,9 @@ jobs: - build_fbtool steps: - name: packaging - run: | - echo "packaging the app now" - cd fbCollector\dist - copy ../ukmda.ico . - copy ../config.ini.sample . - copy ../download_events.sh . - copy ../noimage.jpg . - copy ../README.md . - compress-archive -path . -update -destinationpath c:\temp\fbcollector.zip + uses: Minionguyjpro/Inno-Setup-Action@v1.2.2 + with: + path: .\fbCollector\fireballcollector_setup.iss release_fbtool: name: release fbcollector runs-on: win10 @@ -115,11 +106,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.MM_PAT }} with: - file: c:\temp\fbcollector.zip + file: e:\temp\setup_fireballCollector.exe tags: true draft: true overwrite: true - tag_name: 2025.10.1 + tag_name: 2025.11.1 default_release_name: Tools to Gather Fireball data and Manage UKMON Cameras default_release_body_path: fbCollector\README.md cleanup_fbtool: @@ -131,4 +122,4 @@ jobs: steps: - name: delete package run: | - del c:\temp\fbcollector.zip + del e:\temp\setup_fireballCollector.exe diff --git a/.gitignore b/.gitignore index 6e4b5f8b..e897bccc 100644 --- a/.gitignore +++ b/.gitignore @@ -617,3 +617,4 @@ servercopybkp/* fbcollector/config.ini usermgmt/windows/stationmaint.ini usermgmt/windows/README.md +fbCollector/README.html diff --git a/fbCollector/README.md b/fbCollector/README.md index 4887015b..6cd3c759 100644 --- a/fbCollector/README.md +++ b/fbCollector/README.md @@ -6,43 +6,52 @@ This tool allows authorised users to collect fireball data from UKMON and GMN an * You have installed [WMPL](https://github.com/wmpg/WesternMeteorPyLib/), used to solve trajectories. * You have installed [RMS](https://github.com/CroatianMeteorNetwork/RMS), used to reduce raw data. -* A local folder where you will store fireball data. This is called `basedir` in this documentation. +* You have a local folder where you will store fireball data. This is called `basedir` in this documentation. The above are sufficient to collect data from UKMON and to analyse data either collected from UKMON or by other means. There are additional configuration options that can be used to collect data from GMN and upload solutions to UKMON. These are described inline below. ## Installation -* Clone this repository to a location of your choice -* Install WMPL and RMS, and activate the WMPL python virtual environment. -* Change directory into the location of this code then install the additional requiremnts with `pip install -r requirements.txt` -* Copy `config.ini.sample` to `config.ini` and update the values of `basedir`, `rms_loc`, `rms_env`, `wmpl_loc`, and `wmpl_env` as appropriate. +* Install the package `setup_fireballCollector.exe` from [here](https://github.com/ukmda/ukmda-dataprocessing/releases). +* Install WMPL and RMS and verify they're working. + +## Configuring the App +* Launch the app from the desktop icon or start menu. After a few seconds a text editor will appear to allow you to configure the application. +* Update the values of `basedir`, `rms_loc`, `rms_env`, `wmpl_loc`, and `wmpl_env` as appropriate and save. ## Using the App -* Launch the app by running `fbCollector.ps1` in a Powershell window. After a few seconds the GUI window should appear. -* In the box labelled `Image Selection` enter a date and time in the format `YYYYMMDD_HHMMSS`, then click "Get Images". -* After a few seconds, the listbox below should be populated with images from around the time you selected, provided the UKMON live feed captured something. - * If nothing appears, check the UKMON [livestream](https://archive.ukmeteors.co.uk/live/index.html) to make sure you chose the correct time. +* Enter a date and time in the format `YYYYMMDD_HHMMSS` into the `Image Selection` box, then click "Get Images". + * After a few seconds, the listbox below should be populated with images from around the time you selected, provided the UKMON live feed captured something. + * If nothing appears, check the UKMON [livestream](https://archive.ukmeteors.co.uk/live/index.html) to make sure you chose a suitable time. -* Go through the image list and click `Remove` to delete any that are not of the event you are interested in. +* Go through the image list and click `Remove` or press the `Delete` key to delete any that are not of the event you are interested in. * Once you've whittled the list down to just the interesting events, you can select `Get ECSVs` from the `Raw` menu. This will attempt to get raw data for each image. You can also click `Get Videos` to collect any video data thats available. + * Now you can click `Solve` from the `Solve` menu. This will invoke WMPL and will attempt to find a trajectory that matches the raw data. It may take some time. * If the solver is successful, you can view the solution by selecting `View Solution` from the `Solve` menu. The left-hand list will now show the output of the solver so you can examine it. You can switch back to viewing the raw images by selecting `Review Images` from the `Review` menu. + * If the solve process fails or if the solution seems very poor then try excluding some detections. To do this, select `Excl/Incl ECSV` from the `Raw` menu then rerun the Solver. * If the solution was bad its also worth deleting it before attempting a rerun via `Delete Solution` from the `Solve` menu. ## Manually Reducing an Image -If you've installed RMS then if you have a FITS or FR file from RMS along with the camera's config and platepar files, you can run RMS's `SkyFit2` tool to analyse the image - select and image then select `Reduce Selected Image` from the `Raw` menu. +If you've installed RMS and you have a FITS or FR file from RMS along with the camera's config and platepar files, you can reduce the data using RMS's `SkyFit2` tool. Select an image in the file list then select `Reduce Selected Image` from the `Raw` menu. ## Sending Solutions to UKMON -Once you have a solution, select `Upload orbit` from the `Solve` menu. This will create a Zip file that bundles the created trajectory pickle file with any images and videos. You can then upload the file to Dropbox, Google Drive or another file-sharing site and email a link to fireballdata@ukmeteornetwork.org where one of our team will validate and upload it to our Archive. +Once you have a solution, select `Upload orbit` from the `Solve` menu. This will create a Zip file that bundles the created trajectory pickle file with any images and videos. You can then upload the file to Dropbox, Google Drive or another file-sharing site and email a link to [us](fireballdata@ukmeteornetwork.org) where one of our team will check and upload it to our Archive. #### API Key -Members of the UKMON team who frequently create solutions can request an API key to upload solutions less than 10 MB in size. We'll contact you if we think this is applicable. +Members of the UKMON team who frequently create solutions can request an API key to directly upload solutions less than 10 MB in size. We'll contact you if we think this is applicable. + +## Sharing Raw Data +If you've configured a raw data location in the configuration file, `Share Raw Data` on the raw menu will create a zip file of all the raw data and copy it to the location. For example, if you have Dropbox installed and set the share location to a folder in your Dropbox, then the zip file will be copied there. ### Collecting data from GMN -Members of the GMN coordinators group who have permission from Denis Vida can use this tool to collect data directly from GMN or using the GMN Watchlist. If you fall into this category you can fill in the `[gmnanalysis]` section of the config file with the name of your SSH private key and other details. This will activate additional menu options to `Get GMN Raw Data` and to use the `Watchlist`. -If you're running on Windows, you will need WSL2 enabled and the rsync tool installed in WSL2. +Members of the GMN Coordinators group who have permission from Denis Vida can use this tool to collect raw data directly from GMN. + +If you fall into this category you can fill in the `[gmn]` section of the config file with the name of your SSH private key and other details. This will activate additional menu options to `Get GMN Raw Data` and to use the `Watchlist`. + +*Note that you will need WSL2 enabled and the rsync tool installed in WSL2 to run this process.* ## Logs -The programme creates logs in your system's `TEMP` folder. +You can view the logs from the `File` menu. The logs are created in [%TEMP%/fbcollector](%temp%/fbcollector). diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 242a093f..00d7c92f 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -17,6 +17,7 @@ from PIL import Image import requests import pandas as pd +import webbrowser import paramiko from scp import SCPClient @@ -254,7 +255,7 @@ def readConfig(self): self.gmn_key = None self.gmn_user = None self.gmn_server = None - if localcfg.has_option('gmnkey','gmnkey'): + if localcfg.has_option('gmn','gmnkey'): self.gmn_key = localcfg['gmn']['gmnkey'] self.gmn_user = localcfg['gmn']['gmnuser'] self.gmn_server = localcfg['gmn']['gmnserver'] @@ -283,7 +284,7 @@ def readConfig(self): self.share_loc = None - if localcfg.has_option('share','shrfldr'): + if localcfg.has_option('sharing','shrfldr'): self.share_loc = os.path.expanduser(localcfg['sharing']['shrfldr'].replace('$HOME','~')).replace('\\','/') def quitApplication(self): @@ -418,10 +419,13 @@ def initUI(self): otherMenu = Menu(self.menuBar, tearoff=0) otherMenu.add_command(label="Get Traj Pickle", command=self.getTrajpickle) otherMenu.add_command(label="Add Image/Vid", command=self.addImageVideo) - solveMenu.add_separator() + otherMenu.add_separator() otherMenu.add_command(label="Delete Orbit", command=self.delOrbit) self.menuBar.add_cascade(label="Other", underline=0, menu=otherMenu) + helpMenu = Menu(self.menuBar, tearoff=0) + helpMenu.add_command(label="Documentation", command=self.showDocs) + self.menuBar.add_cascade(label="Help", underline=0, menu=helpMenu) # buttons self.save_panel = LabelFrame(self, text=' Image Selection ') self.save_panel.grid(row = 1, columnspan = 2, sticky='WE') @@ -437,7 +441,6 @@ def initUI(self): save_bmp = StyledButton(self.save_panel, text="Remove", width = 8, command = lambda: self.remove_image()) save_bmp.grid(row = 1, column = 4) - # Listbox self.scrollbar = Scrollbar(self) self.listbox = Listbox(self, width = 47, yscrollcommand=self.scrollbar.set, exportselection=0, activestyle = "none", bg = global_bg, fg = global_fg) @@ -448,6 +451,9 @@ def initUI(self): self.listbox.bind('<>', self.update_image) self.scrollbar.config(command = self.listbox.yview) + # bind delete key + self.listbox.bind("", lambda x: self.remove_image()) + # IMAGE try: # Show the TV test card image on program start @@ -470,6 +476,10 @@ def reviewConfig(self): self.readConfig() self.initUI() + def showDocs(self): + webbrowser.open('README.html') + return + def reduceCamera(self): current_image = self.listbox.get(ACTIVE) if current_image == '': @@ -748,12 +758,13 @@ def archiveFolder(self): log.warning(f'unable to create archive {self.dir_path}') log.warning(e) try: - os.chdir(self.fb_dir) shutil.rmtree(self.dir_path) except Exception as e: log.warning(f'unable to remove folder, please do it manually {self.dir_path}') log.warning(e) self.dir_path = self.fb_dir + self.dir_path = self.fb_dir + self.viewData() def delFolder(self): noimgdata = img.open(noimg_file).resize((640,360)) @@ -796,6 +807,7 @@ def remove_image(self): """ Remove the selected image from disk """ current_image = self.listbox.get(ACTIVE) + curr_row = self.listbox.curselection() if current_image == '': return if not tkMessageBox.askyesno("Delete file", f"delete {current_image}?"): @@ -815,6 +827,11 @@ def remove_image(self): pass self.selected[current_image] = (0,'') self.update_listbox(self.get_bin_list()) + maxrows = self.listbox.index("end") + curr_row = (max(min(maxrows-1, curr_row[0]),0),) + self.listbox.select_set(curr_row) + self.listbox.activate(curr_row) + self.update_image('x') def update_image(self, thing): """ When selected, load a new image @@ -868,14 +885,23 @@ def getData(self): self.update_listbox(self.get_bin_list()) def getTrajpickle(self): - basepatt = os.path.split(self.dir_path)[1] - ymd = basepatt[:8] + if self.dir_path: + basepatt = os.path.split(self.dir_path)[1] + else: + basepatt = '' fullpatt = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue=basepatt) if not fullpatt: return - trajpick = f'{basepatt}_trajectory.pickle' + ymd = fullpatt[:8] + trajpick = f'{fullpatt[:15]}_trajectory.pickle' url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd}/{fullpatt}/{trajpick}' log.info(f'{url}') + if self.patt is None: + self.dir_path = os.path.join(self.fb_dir, fullpatt[:15]) + else: + self.dir_path = os.path.join(self.fb_dir, self.patt) + log.info(self.dir_path) + os.makedirs(self.dir_path, exist_ok=True) get_response = requests.get(url, stream=True) if get_response.status_code == 200: log.info(f'retrieved {trajpick}') @@ -883,6 +909,9 @@ def getTrajpickle(self): for chunk in get_response.iter_content(chunk_size=4096): if chunk: # filter out keep-alive new chunks f.write(chunk) + if self.patt is None: + self.patt = fullpatt[:15] + self.newpatt.set(self.patt) else: log.info(f'unable to retrieve {trajpick}, {get_response.status_code}') return @@ -1040,7 +1069,6 @@ def getEventData(self): ret = tkMessageBox.askyesno("Wait", f'Should wait till {self.evtMonTriggered.strftime("%H:%M:%S")} - continue anyway?') if ret is False: return - os.chdir(self.fb_dir) log.info(f'getting data for {evtdate}') procid = subprocess.Popen(('bash','-c', cmd)) procid.wait() diff --git a/fbCollector/fireballcollector_setup.iss b/fbCollector/fireballcollector_setup.iss new file mode 100644 index 00000000..d9cafc18 --- /dev/null +++ b/fbCollector/fireballcollector_setup.iss @@ -0,0 +1,31 @@ +#define MyAppName "UKMON Fireball Analyser" + +[Setup] +AppName=UKMON Fireball Analyser +AppVersion="1.0.0" +AppPublisher=UK Meteor Network +AppPublisherURL=https://www.ukmeteornetwork.org +DefaultDirName={localappdata}\programs\fbAnalyser +DefaultGroupName=UKMON +UninstallDisplayIcon={app}\fireballCollector.exe +Compression=lzma2 +SolidCompression=yes +OutputDir=e:\temp +OutputBaseFilename=setup_fireballCollector +PrivilegesRequired=lowest + +[Files] +Source: ".\dist\fireballCollector.exe"; DestDir: "{app}" +Source: ".\noimage.jpg"; DestDir: "{app}" +Source: ".\ukmda.ico"; DestDir: "{app}" +Source: ".\download_events.sh"; DestDir: "{app}" +Source: ".\config.ini.sample"; DestDir: "{app}"; Permissions: users-modify +Source: ".\README.html"; DestDir: "{app}" + +[Tasks] +Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; + +[Icons] +Name: "{group}\Fireball Analyser"; Filename: "{app}\fireballCollector.exe"; IconFilename: "{app}\ukmda.ico" +Name: "{group}\{cm:UninstallProgram,{#MyAppName}}"; Filename: "{uninstallexe}" +Name: "{userdesktop}\{#MyAppName}"; Filename: "{app}\fireballCollector.exe"; Tasks: desktopicon; WorkingDir: {app}; IconFilename: "{app}\ukmda.ico" diff --git a/fbCollector/requirements.txt b/fbCollector/requirements.txt index f8283c73..12447143 100644 --- a/fbCollector/requirements.txt +++ b/fbCollector/requirements.txt @@ -6,4 +6,5 @@ scp pyqtgraph xmltodict requests -pandas \ No newline at end of file +pandas +markdown2 \ No newline at end of file diff --git a/usermgmt/windows/cam_management_setup.iss b/usermgmt/windows/cam_management_setup.iss new file mode 100644 index 00000000..bb9adfa7 --- /dev/null +++ b/usermgmt/windows/cam_management_setup.iss @@ -0,0 +1,29 @@ +#define MyAppName "UKMON Camera Management" + +[Setup] +AppName=UKMON Camera Management +AppVersion="2.0.0" +AppPublisher=UK Meteor Network +AppPublisherURL=https://www.ukmeteornetwork.org +DefaultDirName={localappdata}\programs\cameraMgmt +DefaultGroupName=UKMON +UninstallDisplayIcon={app}\stationMaint2.exe +Compression=lzma2 +SolidCompression=yes +OutputDir=e:\temp +OutputBaseFilename=setup_cameraMgmt +PrivilegesRequired=lowest + +[Files] +Source: ".\dist\stationMaint2.exe"; DestDir: "{app}" +Source: ".\camera.ico"; DestDir: "{app}" +Source: ".\stationmaint.ini.sample"; DestDir: "{app}"; Permissions: users-modify +Source: "..\README.md"; DestDir: "{app}"; Permissions: users-modify + +[Tasks] +Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; + +[Icons] +Name: "{group}\Camera Management"; Filename: "{app}\stationMaint2.exe"; IconFilename: "{app}\camera.ico" +Name: "{group}\{cm:UninstallProgram,{#MyAppName}}"; Filename: "{uninstallexe}" +Name: "{userdesktop}\{#MyAppName}"; Filename: "{app}\stationMaint2.exe"; Tasks: desktopicon; WorkingDir: {app}; IconFilename: "{app}\camera.ico"