diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..8fe69c6 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,129 @@ +name: Docker Image CI + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + + docker-build-db: + runs-on: ubuntu-latest + steps: + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner}} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: lowercase github.repository + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV} + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: trackdirect-db.dockerfile + push: true + tags: | + ghcr.io/${{ env.IMAGE_NAME }}-db:latest + ghcr.io/${{ env.IMAGE_NAME }}-db:${{ github.RUN_ID }} + + docker-build-apache: + runs-on: ubuntu-latest + steps: + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner}} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: lowercase github.repository + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV} + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: trackdirect-apache.dockerfile + push: true + tags: | + ghcr.io/${{ env.IMAGE_NAME }}-apache:latest + ghcr.io/${{ env.IMAGE_NAME }}-apache:${{ github.RUN_ID }} + + docker-build-python: + runs-on: ubuntu-latest + steps: + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner}} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: lowercase github.repository + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV} + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: trackdirect-python.dockerfile + push: true + tags: | + ghcr.io/${{ env.IMAGE_NAME }}-python:latest + ghcr.io/${{ env.IMAGE_NAME }}-python:${{ github.RUN_ID }} + + docker-build-aprsc: + runs-on: ubuntu-latest + steps: + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner}} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: lowercase github.repository + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV} + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: trackdirect-aprsc.dockerfile + push: true + tags: | + ghcr.io/${{ env.IMAGE_NAME }}-aprsc:latest + ghcr.io/${{ env.IMAGE_NAME }}-aprsc:${{ github.RUN_ID }} + + docker-build-cron: + runs-on: ubuntu-latest + steps: + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner}} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: lowercase github.repository + run: echo "IMAGE_NAME=${GITHUB_REPOSITORY@L}" >> ${GITHUB_ENV} + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: trackdirect-cron.dockerfile + push: true + tags: | + ghcr.io/${{ env.IMAGE_NAME }}-cron:latest + ghcr.io/${{ env.IMAGE_NAME }}-cron:${{ github.RUN_ID }} diff --git a/crontab b/crontab index 3ee247f..db3694e 100644 --- a/crontab +++ b/crontab @@ -1,2 +1,2 @@ 40 * * * * ~/trackdirect/server/scripts/remover.sh trackdirect.ini -*/30 * * * * ~/trackdirect/server/scripts/ogn_devices_install.sh trackdirect db 5432 postgres +*/30 * * * * ~/trackdirect/server/scripts/ogn_devices_install.sh trackdirect.ini diff --git a/htdocs/public/index.php b/htdocs/public/index.php index 1d2f4c0..da8d809 100755 --- a/htdocs/public/index.php +++ b/htdocs/public/index.php @@ -45,7 +45,7 @@ - + diff --git a/server/bin/remover.py b/server/bin/remover.py index c7637d0..bf73b0e 100644 --- a/server/bin/remover.py +++ b/server/bin/remover.py @@ -27,8 +27,8 @@ def validate_config_file(config_file): print("\nUsage: script.py [config.ini]") sys.exit() -def drop_table_if_exists(cursor, table_name, logger): - if DatabaseObjectFinder.check_table_exists(table_name): +def drop_table_if_exists(cursor, dbobjfinder, table_name, logger): + if dbobjfinder.check_table_exists(table_name): cursor.execute(f"DROP TABLE {table_name}") logger.info(f"Dropped table {table_name}") @@ -52,11 +52,21 @@ def main(): max_days_to_save_weather_data = config.days_to_save_weather_data max_days_to_save_telemetry_data = config.days_to_save_telemetry_data + # Use the highest retention across all data types so drop loops cover everything + max_retention = max( + max_days_to_save_position_data, + max_days_to_save_weather_data, + max_days_to_save_telemetry_data + ) + # Scan far enough ahead to handle long cron outages (up to ~1 year) + drop_lookahead = 365 + logger.info("Starting") logger.info(f"Saving position data for {max_days_to_save_position_data} days") logger.info(f"Saving station data for {max_days_to_save_station_data} days") logger.info(f"Saving weather data for {max_days_to_save_weather_data} days") logger.info(f"Saving telemetry data for {max_days_to_save_telemetry_data} days") + logger.info(f"Drop lookahead set to {drop_lookahead} days (max retention = {max_retention})") try: track_direct_db = DatabaseConnection() @@ -69,56 +79,62 @@ def main(): track_direct_db_object_finder = DatabaseObjectFinder(db) packet_repository = PacketRepository(db) - # Loop over the latest days and delete packets that are not needed anymore - for x in range(2, 16): + for x in range(2, max_days_to_save_position_data + 1): prev_day = datetime.date.today() - datetime.timedelta(x) prev_day_timestamp = int(prev_day.strftime("%s")) - prev_day_format = datetime.datetime.utcfromtimestamp(prev_day_timestamp).strftime('%Y%m%d') + prev_day_format = datetime.datetime.fromtimestamp(prev_day_timestamp, datetime.timezone.utc).strftime('%Y%m%d') packet_table = f"packet{prev_day_format}" if track_direct_db_object_finder.check_table_exists(packet_table): - deleted_rows = None - do_full_vacuum = False - while deleted_rows is None or deleted_rows >= 5000: - sql = f"""DELETE FROM {packet_table} WHERE id IN ( - SELECT id FROM {packet_table} WHERE map_id NOT IN (1,12) LIMIT 5000)""" - cursor.execute(sql) - deleted_rows = cursor.rowcount - logger.info(f"Deleted {deleted_rows} from {packet_table}") - if deleted_rows > 0: - do_full_vacuum = True - time.sleep(0.5) - if do_full_vacuum: - cursor.execute(f"VACUUM FULL {packet_table}_path") - cursor.execute(f"REINDEX TABLE {packet_table}_path") - cursor.execute(f"VACUUM FULL {packet_table}") - cursor.execute(f"REINDEX TABLE {packet_table}") - - # Drop packet_weather - for x in range(max_days_to_save_weather_data, max_days_to_save_weather_data + 100): - prev_day = datetime.date.today() - datetime.timedelta(x) - prev_day_format = prev_day.strftime('%Y%m%d') - packet_table = f"packet{prev_day_format}_weather" - drop_table_if_exists(cursor, packet_table, logger) - - # Drop packet_telemetry - for x in range(max_days_to_save_telemetry_data, max_days_to_save_telemetry_data + 100): + # Count rows to delete first + cursor.execute(f"SELECT count(*) FROM {packet_table} WHERE map_id NOT IN (1,12)") + count = cursor.fetchone()[0] + if count == 0: + logger.info(f"No stale rows in {packet_table}, skipping") + continue + + logger.info(f"Deleting {count} stale rows from {packet_table}") + cursor.execute(f"DELETE FROM {packet_table} WHERE map_id NOT IN (1,12)") + deleted_rows = cursor.rowcount + logger.info(f"Deleted {deleted_rows} from {packet_table}") + + if deleted_rows > 0: + logger.info(f"Cleanup deferred to autovacuum for {packet_table}") + + scanned_days = 0 + dropped_tables = 0 + for x in range(max_retention, max_retention + drop_lookahead + 1): + scanned_days += 1 prev_day = datetime.date.today() - datetime.timedelta(x) prev_day_format = prev_day.strftime('%Y%m%d') - packet_table = f"packet{prev_day_format}_telemetry" - drop_table_if_exists(cursor, packet_table, logger) - - # Drop packets - for x in range(max_days_to_save_position_data, max_days_to_save_position_data + 100): - prev_day = datetime.date.today() - datetime.timedelta(x) - prev_day_format = prev_day.strftime('%Y%m%d') - packet_table = f"packet{prev_day_format}" - - drop_table_if_exists(cursor, f"{packet_table}_ogn", logger) - drop_table_if_exists(cursor, f"{packet_table}_path", logger) - drop_table_if_exists(cursor, packet_table, logger) + base_table = f"packet{prev_day_format}" + + # Drop weather sub-table (only if past weather retention) + if x >= max_days_to_save_weather_data: + if track_direct_db_object_finder.check_table_exists(f"{base_table}_weather"): + dropped_tables += 1 + drop_table_if_exists(cursor, track_direct_db_object_finder, f"{base_table}_weather", logger) + + # Drop telemetry sub-table (only if past telemetry retention) + if x >= max_days_to_save_telemetry_data: + if track_direct_db_object_finder.check_table_exists(f"{base_table}_telemetry"): + dropped_tables += 1 + drop_table_if_exists(cursor, track_direct_db_object_finder, f"{base_table}_telemetry", logger) + + # Drop position sub-tables and main table (only if past position retention) + if x >= max_days_to_save_position_data: + if track_direct_db_object_finder.check_table_exists(f"{base_table}_ogn"): + dropped_tables += 1 + drop_table_if_exists(cursor, track_direct_db_object_finder, f"{base_table}_ogn", logger) + if track_direct_db_object_finder.check_table_exists(f"{base_table}_path"): + dropped_tables += 1 + drop_table_if_exists(cursor, track_direct_db_object_finder, f"{base_table}_path", logger) + if track_direct_db_object_finder.check_table_exists(base_table): + dropped_tables += 1 + drop_table_if_exists(cursor, track_direct_db_object_finder, base_table, logger) + + logger.info(f"Phase 2 complete: scanned {scanned_days} days, dropped {dropped_tables} tables") - # Delete old stations timestamp_limit = int(time.time()) - (60 * 60 * 24 * max_days_to_save_station_data) deleted_rows = 0 sql = """SELECT station.id, station.latest_sender_id, station.name @@ -148,7 +164,7 @@ def main(): logger.info(f"Trying to delete station {record['name']} ({record['id']})") delete_cursor = db_no_auto_commit.cursor() try: - for table in ['station_telemetry_bits', 'station_telemetry_eqns', 'station_telemetry_param', 'station_telemetry_unit', 'station_city']: + for table in ['station_telemetry_bits', 'station_telemetry_eqns', 'station_telemetry_param', 'station_telemetry_unit']: delete_cursor.execute(f"DELETE FROM {table} WHERE station_id = %s", (record["id"],)) delete_cursor.execute("DELETE FROM station WHERE id = %s", (record["id"],)) @@ -157,7 +173,6 @@ def main(): db_no_auto_commit.commit() delete_cursor.close() deleted_rows += 1 - time.sleep(0.5) except Exception as e: logger.error(e, exc_info=1) db_no_auto_commit.rollback() @@ -172,7 +187,6 @@ def main(): cursor.execute("VACUUM ANALYZE sender") cursor.execute("REINDEX TABLE sender") - # Close DB connection cursor.close() db.close() logger.info("Done!") diff --git a/server/scripts/ogn_devices_install.sh b/server/scripts/ogn_devices_install.sh index 2175b76..8caab7e 100755 --- a/server/scripts/ogn_devices_install.sh +++ b/server/scripts/ogn_devices_install.sh @@ -2,14 +2,29 @@ if [ $# -eq 0 ] then echo "No arguments supplied" - echo "$0 [dbname] [dbport] [dbuser]" + echo "$0 [config file]" exit fi -DATABASE=$1 -HOST=$2 -PORT=$3 -USER=$4 +CONFIGFILE=$1 +if [[ "$CONFIGFILE" != /* ]]; then + CONFIGFILE="$HOME/trackdirect/config/$CONFIGFILE" +fi + +if [ ! -f "$CONFIGFILE" ]; then + echo "Config file $CONFIGFILE does not exist" + exit +fi + +# Parse database credentials from the .ini config file +DB_HOST=$(grep -oP '^host\s*=\s*\K.*' "$CONFIGFILE" | head -1) +DB_PORT=$(grep -oP '^port\s*=\s*\K.*' "$CONFIGFILE" | head -1) +DB_NAME=$(grep -oP '^database\s*=\s*\K.*' "$CONFIGFILE" | head -1) +DB_USER=$(grep -oP '^username\s*=\s*\K.*' "$CONFIGFILE" | head -1) +DB_PASS=$(grep -oP '^password\s*=\s*\K.*' "$CONFIGFILE" | head -1) + +# Export password so psql doesn't need .pgpass +export PGPASSWORD="$DB_PASS" pushd `dirname $0` > /dev/null SCRIPTPATH=`pwd -P` @@ -17,10 +32,10 @@ popd > /dev/null # Create dir and remove old stuff (keep zip-file since it may be equal to latest) mkdir -p $SCRIPTPATH/ogndevices -mkdir -p $SCRIPTPATH/ogndevices/${DATABASE} -rm $SCRIPTPATH/ogndevices/${DATABASE}/*.csv -rm $SCRIPTPATH/ogndevices/${DATABASE}/*.txt -cd $SCRIPTPATH/ogndevices/${DATABASE} +mkdir -p $SCRIPTPATH/ogndevices/${DB_NAME} +rm $SCRIPTPATH/ogndevices/${DB_NAME}/*.csv +rm $SCRIPTPATH/ogndevices/${DB_NAME}/*.txt +cd $SCRIPTPATH/ogndevices/${DB_NAME} # Download latest csv file (but only if newer) wget -N http://ddb.glidernet.org/download/?t=1 -O ogndevices.csv @@ -34,8 +49,8 @@ else # Remove comments in file sed '/^#/ d' < ogndevices.csv > ogndevices2.csv -# Load file into database (assumes .pgpass is correctly set) -psql -h $HOST -p $PORT $DATABASE -U $USER << EOF +# Load file into database (password comes from PGPASSWORD env var) +psql -h $DB_HOST -p $DB_PORT $DB_NAME -U $DB_USER << EOF create table if not exists ogn_device ( "device_type" text not null, @@ -52,7 +67,7 @@ begin transaction; drop index if exists ogn_device_device_id_idx; truncate ogn_device; -\copy ogn_device from '$SCRIPTPATH/ogndevices/$DATABASE/ogndevices2.csv' DELIMITERS ',' CSV QUOTE ''''; +\copy ogn_device from '$SCRIPTPATH/ogndevices/$DB_NAME/ogndevices2.csv' DELIMITERS ',' CSV QUOTE ''''; create index ogn_device_device_id_idx on ogn_device(device_id); insert into ogn_device(device_type, device_id, aircraft_model, registration, cn, tracked, identified, ddb_aircraft_type) values ('F', '3FEF6F', '', '', '', 'N', 'N', 1); @@ -62,4 +77,4 @@ EOF fi -exit 0 +exit 0 \ No newline at end of file diff --git a/trackdirect-apache.dockerfile b/trackdirect-apache.dockerfile index ee1a2d4..d86844a 100644 --- a/trackdirect-apache.dockerfile +++ b/trackdirect-apache.dockerfile @@ -1,4 +1,4 @@ -FROM php:8.2-apache +FROM php:8.4.8-apache RUN apt-get update && apt-get install -y \ git \ diff --git a/trackdirect-aprsc.dockerfile b/trackdirect-aprsc.dockerfile index 767b287..ed4580c 100644 --- a/trackdirect-aprsc.dockerfile +++ b/trackdirect-aprsc.dockerfile @@ -4,7 +4,7 @@ RUN apt-get update && apt-get install -y \ gnupg \ && rm -rf /var/lib/apt/lists/* -RUN printf "deb http://aprsc-dist.he.fi/aprsc/apt noble main" >> /etc/apt/sources.list +RUN printf "deb [trusted=yes] http://aprsc-dist.he.fi/aprsc/apt noble main" >> /etc/apt/sources.list RUN gpg --keyserver keyserver.ubuntu.com --recv C51AA22389B5B74C3896EF3CA72A581E657A2B8D RUN gpg --export C51AA22389B5B74C3896EF3CA72A581E657A2B8D | apt-key add - diff --git a/trackdirect-db.dockerfile b/trackdirect-db.dockerfile index a9c9237..e0576f8 100644 --- a/trackdirect-db.dockerfile +++ b/trackdirect-db.dockerfile @@ -1,7 +1,7 @@ -FROM postgres +FROM postgres:17 COPY misc/database/tables/* /docker-entrypoint-initdb.d/ COPY config/postgresql.conf /etc/postgresql.conf RUN chown :999 /etc/postgresql.conf RUN chmod 770 /etc/postgresql.conf RUN chmod g+s /etc/postgresql.conf -VOLUME /var/lib/postgresql/data \ No newline at end of file +VOLUME /var/lib/postgresql/data