diff --git a/.github/workflows/deploy.staging.yml b/.github/workflows/deploy.staging.yml new file mode 100644 index 0000000..ff3fed9 --- /dev/null +++ b/.github/workflows/deploy.staging.yml @@ -0,0 +1,27 @@ +name: Build and Deploy (staging) + +on: + push: + branches: + - staging + +jobs: + deploy: + runs-on: ubuntu-latest + environment: + name: staging + + permissions: + contents: read + + steps: + - name: Deploy via SSH + uses: appleboy/ssh-action@v1.2.5 + with: + host: ${{ secrets.HOST }} + username: ${{ secrets.USER }} + key: ${{ secrets.SSH_KEY }} + envs: PATH + script: | + set -e + sudo /opt/relaysms/relaysms-gateway-server/manage.sh update diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml deleted file mode 100644 index c99ac27..0000000 --- a/.github/workflows/staging-deploy.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Staging Server Build Pipeline - -on: - push: - branches: - - staging - -jobs: - deploy: - name: Deploy to Staging Server - runs-on: ubuntu-latest - environment: - name: staging - steps: - - name: Execute Remote SSH Commands - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.HOST }} - username: ${{ secrets.USERNAME }} - key: ${{ secrets.KEY }} - script: | - set -e - - echo "============================" - echo "Updating repository ..." - echo "============================" - if ! assembler clone --branch staging --project gateway_server; then - echo "❌ Error updating repository!" - exit 1 - fi - echo "===============================" - echo "✅ Repository update complete" - echo "===============================" - - echo "=========================" - echo "Building project ..." - echo "=========================" - if ! assembler deploy --project gateway_server; then - echo "❌ Error building project!" - exit 1 - fi - echo "===========================" - echo "✅ Project build complete" - echo "===========================" - - echo "=============================" - echo "Cleaning up staging builds ..." - echo "=============================" - if ! ${{ secrets.CLEANUP_CMD }}; then - echo "❌ Error cleaning up builds!" - exit 1 - fi - echo "=============================" - echo "✅ Cleanup complete" - echo "=============================" diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..7303c1d --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,271 @@ +# Installation Guide + +## Automated Installation + +```bash +sudo ./install.sh +``` + +This will: + +- Install system dependencies +- Clone repository to `/opt/relaysms/relaysms-gateway-server` +- Setup Python virtualenv +- Compile gRPC protos +- Install and enable systemd services + +## Manual Installation + +### Install Dependencies + +```bash +sudo apt update +sudo apt install -y python3 python3-pip python3-venv python3-dev \ + libmariadb-dev git curl make +``` + +### Clone Repository + +```bash +sudo git clone https://github.com/smswithoutborders/RelaySMS-Gateway-Server.git \ + /opt/relaysms/relaysms-gateway-server +cd /opt/relaysms/relaysms-gateway-server +``` + +### Setup Python Environment + +```bash +python3 -m venv venv +venv/bin/pip install --upgrade pip +venv/bin/pip install -r requirements.txt +``` + +### Build Application + +```bash +source venv/bin/activate +make grpc-compile +``` + +This will: + +- Download publisher and bridge proto files +- Compile gRPC protos + +### Configure Environment + +```bash +cp template.env .env +vim .env +``` + +Edit the `.env` file to configure: + +- Database settings (MySQL or SQLite) +- Publisher gRPC connection settings +- Bridge gRPC connection settings +- Server ports and hosts +- IMAP settings for email monitoring +- FTP server settings + +### Initialize Runtime + +```bash +mkdir -p data/ftp_file_store +set -a && source .env && set +a +# Database will be created automatically on first run +``` + +### Install Services + +```bash +sudo cp relaysms-gateway-server.target relaysms-gateway-server-*.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable relaysms-gateway-server.target +sudo systemctl start relaysms-gateway-server.target +``` + +## Service Management + +```bash +./manage.sh start # Start all services +./manage.sh stop # Stop all services +./manage.sh restart # Restart all services +./manage.sh status # Check status +./manage.sh logs # View logs +./manage.sh enable # Enable on boot +./manage.sh disable # Disable on boot +./manage.sh update # Update installation +./manage.sh uninstall # Remove installation +``` + +## Configuration + +Edit `/opt/relaysms/relaysms-gateway-server/.env`: + +### Server + +Configure REST API server hosts and ports: + +```bash +# REST API +HOST=127.0.0.1 +PORT=5000 +SSL_PORT=5001 + +# SSL Configuration (optional) +SSL_CERTIFICATE=/path/to/certificate.crt +SSL_KEY=/path/to/private.key +SSL_PEM=/path/to/certificate.pem +SSL_SERVER_NAME=localhost +``` + +### Publisher Connection + +Configure connection to RelaySMS Publisher: + +```bash +PUBLISHER_GRPC_HOST=127.0.0.1 +PUBLISHER_GRPC_PORT=6000 +``` + +### Bridge Connection + +Configure connection to RelaySMS Bridge: + +```bash +BRIDGE_GRPC_HOST=127.0.0.1 +BRIDGE_GRPC_PORT=10000 +``` + +### Database + +Choose between MySQL and SQLite: + +**SQLite (Default):** + +```bash +SQLITE_DATABASE_PATH=data/gateway_server.db +# Leave MySQL settings empty +``` + +**MySQL:** + +```bash +MYSQL_HOST=127.0.0.1 +MYSQL_USER=your_user +MYSQL_PASSWORD=your_password +MYSQL_DATABASE=relaysms_gateway_server +# Leave SQLITE_DATABASE_PATH empty +``` + +### IMAP Configuration + +Configure IMAP settings for email monitoring: + +```bash +IMAP_SERVER=imap.example.com +IMAP_PORT=993 +IMAP_USERNAME=your_email@example.com +IMAP_PASSWORD=your_password +MAIL_FOLDER=INBOX +``` + +### FTP Configuration + +Configure FTP server settings: + +```bash +FTP_USERNAME=your_ftp_user +FTP_PASSWORD=your_ftp_password +FTP_IP_ADDRESS=localhost +FTP_PORT=2222 +FTP_PASSIVE_PORTS=60000-65000 +FTP_READ_LIMIT=51200 +FTP_WRITE_LIMIT=51200 +FTP_MAX_CON=256 +FTP_MAX_CON_PER_IP=5 +FTP_DIRECTORY=data/ftp_file_store +``` + +### Security + +Configure security settings: + +```bash +SMTP_ALLOWED_EMAIL_ADDRESSES=allowed1@example.com,allowed2@example.com +DISABLE_BRIDGE_PAYLOADS_OVER_HTTP=false +``` + +### CORS + +Configure allowed CORS origins: + +```bash +ORIGINS=["http://localhost:3000","https://example.com"] +``` + +## Services + +- `relaysms-gateway-server-rest.service` - REST API (default port 5000, SSL port 5001) +- `relaysms-gateway-server-imap.service` - IMAP Listener for email monitoring +- `relaysms-gateway-server-ftp.service` - FTP Server for file transfers +- `relaysms-gateway-server.target` - Service group + +## File Locations + +- Installation: `/opt/relaysms/relaysms-gateway-server/` +- Configuration: `/opt/relaysms/relaysms-gateway-server/.env` +- Database: `/opt/relaysms/relaysms-gateway-server/data/gateway_server.db` +- FTP Storage: `/opt/relaysms/relaysms-gateway-server/data/ftp_file_store/` +- Service files: `/etc/systemd/system/relaysms-gateway-server*` + +## External Dependencies + +### RelaySMS Publisher (Required) + +The Gateway Server requires a running instance of RelaySMS Publisher for publishing content to online platforms. + +**Installation:** + +See [RelaySMS Publisher Installation Guide](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/INSTALL.md) + +Quick install: + +```bash +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Publisher/main/install.sh | sudo bash +``` + +**Configuration:** + +Ensure the Publisher gRPC server is accessible and update the `PUBLISHER_GRPC_HOST` and `PUBLISHER_GRPC_PORT` variables in the Gateway Server's `.env` file accordingly. + +### RelaySMS Bridge (Required) + +The Gateway Server requires a running instance of RelaySMS Bridge for publishing content to email bridges. + +**Installation:** + +See [RelaySMS Bridge Installation Guide](https://github.com/smswithoutborders/RelaySMS-Bridge-Server/blob/main/INSTALL.md) + +**Configuration:** + +Ensure the Bridge gRPC server is accessible and update the `BRIDGE_GRPC_HOST` and `BRIDGE_GRPC_PORT` variables in the Gateway Server's `.env` file accordingly. + +### IMAP Email Server (Optional) + +If you want to use the IMAP listener feature, you'll need access to an email account with IMAP enabled. + +**Gmail Setup:** + +1. Enable IMAP in Gmail settings +2. Create an App Password: + - Visit: + - Sign in with your Google account + - Select "Mail" as the app and "Other" as the device + - Generate an app password +3. Use the generated app password as `IMAP_PASSWORD` in `.env` + +**Other Email Providers:** + +Consult your email provider's documentation for IMAP server details and authentication requirements. diff --git a/README.md b/README.md index e90e19c..1bcf90a 100644 --- a/README.md +++ b/README.md @@ -4,126 +4,100 @@ RelaySMS Gateway Server is the online router that receives messages from gateway ## Table of Contents -1. [Quick Start](#quick-start) -2. [System Requirements](#system-requirements) -3. [Installation](#installation) -4. [Configuration](#configuration) -5. [References](#references) -6. [Contributing](#contributing) -7. [License](#license) +- [Requirements](#requirements) +- [Installation](#installation) +- [Configuration](#configuration) +- [Docker](#docker) +- [Documentation](#documentation) +- [Contributing](#contributing) +- [License](#license) -## Quick Start +## Requirements -> [!NOTE] -> -> Ensure all [system dependencies](#system-requirements) are installed before running setup scripts. +- **Python:** ≥ 3.8.10 +- **Database:** MySQL (≥ 8.0.28), MariaDB, or SQLite +- **External Services:** + - [RelaySMS Publisher](https://github.com/smswithoutborders/RelaySMS-Publisher) (required) + - [RelaySMS Bridge](https://github.com/smswithoutborders/RelaySMS-Bridge-Server) (required) + - IMAP Email Server (optional, for email monitoring) -For development, use the provided scripts: +**Ubuntu Dependencies:** ```bash -source scripts/quick-setup.sh && ./scripts/quick-start.sh +sudo apt install python3-dev libmysqlclient-dev make ``` -- `quick-setup`: - - - Creates a Python virtual environment (if missing) - - Installs Python dependencies - - Sets up a `.env` file - - Exports environment variables - - Downloads the `publisher` and `bridge` Protobuf files. (via `make publisher-proto` and `make bridge-proto` respectively) - - Compiles gRPC protos (via `make grpc-compile`) - -- `quick-start`: - - Launches the REST server, IMAP Listener, and the FTP Server. - -> [!WARNING] -> -> This setup is for development only. Do not use in production. - -## System Requirements +## Installation -- **Database:** MySQL (≥ 8.0.28), MariaDB, or SQLite -- **Python:** ≥ 3.8.10 -- **Virtual Environments:** Python venv +### Production -### Ubuntu Dependencies +Quick install: ```bash -sudo apt update -sudo apt install python3-dev libmysqlclient-dev apache2 apache2-dev make libapache2-mod-wsgi-py3 +curl -fsSL https://raw.githubusercontent.com/smswithoutborders/RelaySMS-Gateway-Server/main/install.sh | sudo bash ``` -## Installation - -1. **Clone the repository:** +Manage services: - ```bash - git clone https://github.com/smswithoutborders/RelaySMS-Gateway-Server.git - cd RelaySMS-Gateway-Server - ``` +```bash +cd /opt/relaysms/relaysms-gateway-server +./manage.sh {start|stop|restart|status|logs|update} +``` -2. **Create and activate a virtual environment:** +See [INSTALL.md](INSTALL.md) for manual installation and detailed configuration. - ```bash - python3 -m venv venv - source venv/bin/activate - ``` +### Development -3. **Install dependencies:** +```bash +# Setup environment +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt + +# Configure +cp template.env .env +# Edit .env as needed + +# Build +make grpc-compile + +# Start services (use separate terminals) +python3 -m src.imap_listener # Terminal 1 (optional) +python3 -m src.ftp_server # Terminal 2 +make start-rest-api # Terminal 3 +``` - ```bash - pip install -r requirements.txt - ``` +**Quick Development Setup:** -4. **Compile gRPC protos:** +```bash +./scripts/quick-setup.sh && ./scripts/quick-start.sh +``` - ```bash - make grpc-compile - ``` +> [!WARNING] +> Quick setup is for development only. Do not use in production. -## Building and Running with Docker +## Docker -#### Build the Docker Image +### Build the Docker Image ```bash -docker build -t relaysms-gateway-server . +docker build -t relaysms-gateway-server:latest . ``` -#### Run the Container - -> [!TIP] -> -> **For long-term development, you may want to run the container in detached mode (`-d`) and view logs with:** -> -> ```bash -> docker logs -f -> ``` +### Run the Container ```bash -docker run --rm \ +docker run -d \ + --name relaysms-gateway-server \ --env-file .env \ -p 5000:5000 -p 5001:5001 \ -p 2222:2222 -p 60000-65000:60000-65000 \ - -v $(pwd)/ftp_file_store:/gateway_server/ftp_file_store \ - relaysms-gateway-server + -v $(pwd)/data:/gateway_server/data \ + relaysms-gateway-server:latest ``` > [!TIP] -> -> - To run in detached mode: -> ```bash -> docker run -d \ -> --name relaysms-gateway-server \ -> --env-file .env \ -> -p 5000:5000 -p 5001:5001 \ -> -p 2222:2222 -p 60000-65000:60000-65000 \ -> -v $(pwd)/ftp_file_store:/gateway_server/ftp_file_store \ -> relaysms-gateway-server -> ``` -> Then view logs with: -> ```bash -> docker logs -f relaysms-gateway-server -> ``` +> Update `HOST=0.0.0.0` in `.env` for external container access. --- @@ -160,11 +134,17 @@ export PORT=5000 - `PUBLISHER_GRPC_HOST`: Publisher gRPC server host (default: `127.0.0.1`) - `PUBLISHER_GRPC_PORT`: Publisher gRPC server port (default: `6000`) +> [!IMPORTANT] +> RelaySMS Publisher must be installed and running. See [RelaySMS Publisher Installation](https://github.com/smswithoutborders/RelaySMS-Publisher/blob/main/INSTALL.md) + ### Bridge gRPC - `BRIDGE_GRPC_HOST`: Bridge gRPC server host (default: `127.0.0.1`) - `BRIDGE_GRPC_PORT`: Bridge gRPC server port (default: `10000`) +> [!IMPORTANT] +> RelaySMS Bridge must be installed and running. See [RelaySMS Bridge Installation](https://github.com/smswithoutborders/RelaySMS-Bridge-Server/blob/main/INSTALL.md) + ### CORS - `ORIGINS`: Allowed CORS origins (default: `[]`) @@ -175,7 +155,7 @@ export PORT=5000 - `MYSQL_USER`: MySQL username - `MYSQL_PASSWORD`: MySQL password - `MYSQL_DATABASE`: MySQL database (default: `relaysms_gateway_server`) -- `SQLITE_DATABASE_PATH`: SQLite file path (default: `gateway_server.db`) +- `SQLITE_DATABASE_PATH`: SQLite file path (default: `data/gateway_server.db`) ### IMAP Configuration @@ -185,6 +165,11 @@ export PORT=5000 - `IMAP_PASSWORD`: IMAP password - `MAIL_FOLDER`: Mail folder to monitor (default: `INBOX`) +> [!NOTE] +> IMAP configuration is optional. Required only if you want to monitor emails for incoming messages. +> +> **Gmail Setup:** Enable IMAP in settings and create an App Password at . See [INSTALL.md](INSTALL.md#imap-email-server-optional) for details. + ### FTP Configuration - `FTP_USERNAME`: FTP username @@ -196,7 +181,7 @@ export PORT=5000 - `FTP_WRITE_LIMIT`: FTP write limit in bytes (default: `51200`) - `FTP_MAX_CON`: Maximum FTP connections (default: `256`) - `FTP_MAX_CON_PER_IP`: Maximum FTP connections per IP (default: `5`) -- `FTP_DIRECTORY`: FTP directory path (default: `ftp_file_store`) +- `FTP_DIRECTORY`: FTP directory path (default: `data/ftp_file_store`) ### Security @@ -207,11 +192,11 @@ export PORT=5000 - `LOG_LEVEL`: Logging level (default: `info`) -## References +## Documentation -- REST API Resources: - - [API V3](docs/api_v3.md) -- [Gateway Client CLI](docs/gateway_clients_cli.md) +- [Installation Guide](INSTALL.md) - Detailed setup instructions +- [API V3](docs/api_v3.md) - REST API documentation +- [Gateway Client CLI](docs/gateway_clients_cli.md) - CLI documentation ## Contributing diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..b6010c9 --- /dev/null +++ b/install.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +set -e + +INSTALL_DIR="/opt/relaysms/relaysms-gateway-server" +SERVICE_NAME="relaysms-gateway-server" +REPO_URL="https://github.com/smswithoutborders/RelaySMS-Gateway-Server.git" +BRANCH="${BRANCH:-main}" + +log() { echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1"; } +error() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1" >&2 + exit 1 +} +check_root() { [ "$EUID" -eq 0 ] || error "This script must be run with sudo"; } + +install_dependencies() { + log "Installing dependencies" + apt update || error "Failed to update package list" + apt install -y python3 python3-pip python3-venv python3-dev \ + libmariadb-dev git curl make || error "Failed to install dependencies" +} + +clone_repository() { + log "Cloning repository" + if [ -d "$INSTALL_DIR/.git" ]; then + log "Repository exists, updating" + cd "$INSTALL_DIR" + git fetch origin && git checkout "$BRANCH" && git pull origin "$BRANCH" || error "Failed to update" + else + mkdir -p "$(dirname "$INSTALL_DIR")" + git clone -b "$BRANCH" "$REPO_URL" "$INSTALL_DIR" || error "Failed to clone" + fi +} + +setup_virtualenv() { + log "Setting up virtual environment" + cd "$INSTALL_DIR" + [ -d "venv" ] || python3 -m venv venv || error "Failed to create venv" + venv/bin/pip install --upgrade pip || error "Failed to upgrade pip" + venv/bin/pip install -r requirements.txt || error "Failed to install dependencies" +} + +compile_grpc() { + log "Compiling gRPC protos" + make grpc-compile || error "Failed to compile gRPC" +} + +setup_env() { + log "Setting up configuration" + [ -f ".env" ] && log ".env already exists" && return + [ -f "template.env" ] || error "template.env not found" + cp template.env .env +} + +setup_runtime() { + log "Setting up runtime" + mkdir -p data/ftp_file_store + set -a && source .env && set +a +} + +install_systemd_service() { + log "Installing systemd services" + for service in relaysms-gateway-server.target relaysms-gateway-server-rest.service relaysms-gateway-server-imap.service relaysms-gateway-server-ftp.service; do + [ -f "$service" ] || error "Service file $service not found" + cp "$service" /etc/systemd/system/$service || error "Failed to install $service" + done + systemctl daemon-reload || error "Failed to reload systemd" + systemctl enable "$SERVICE_NAME.target" || error "Failed to enable services" + systemctl start "$SERVICE_NAME.target" || error "Failed to start services" +} + +set_permissions() { + log "Setting permissions" + chmod 600 .env 2>/dev/null || true +} + +main() { + log "Starting installation" + + check_root + install_dependencies + clone_repository + setup_virtualenv + + cd "$INSTALL_DIR" + source venv/bin/activate + + compile_grpc + setup_env + setup_runtime + set_permissions + install_systemd_service + + log "Installation complete" + log "" + log "Services started and enabled" + log "Manage: $INSTALL_DIR/manage.sh {start|stop|restart|status|logs}" + log "Config: $INSTALL_DIR/.env" +} + +main "$@" diff --git a/manage.sh b/manage.sh new file mode 100755 index 0000000..48842f5 --- /dev/null +++ b/manage.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +SERVICE_NAME="relaysms-gateway-server.target" +INSTALL_DIR="/opt/relaysms/relaysms-gateway-server" + +check_sudo() { + [ "$EUID" -eq 0 ] || { + echo "Requires sudo" + exit 1 + } +} + +case "$1" in +start) + check_sudo + systemctl start $SERVICE_NAME + echo "Services started" + ;; +stop) + check_sudo + systemctl stop $SERVICE_NAME + echo "Services stopped" + ;; +restart) + check_sudo + systemctl restart $SERVICE_NAME + echo "Services restarted" + ;; +status) + systemctl status relaysms-gateway-server-rest relaysms-gateway-server-imap relaysms-gateway-server-ftp + ;; +logs) + check_sudo + journalctl -u relaysms-gateway-server-rest -u relaysms-gateway-server-imap -u relaysms-gateway-server-ftp -f + ;; +enable) + check_sudo + systemctl enable $SERVICE_NAME + echo "Services enabled on boot" + ;; +disable) + check_sudo + systemctl disable $SERVICE_NAME + echo "Services disabled on boot" + ;; +update) + check_sudo + systemctl stop $SERVICE_NAME + cd "$INSTALL_DIR" + git pull + source venv/bin/activate + pip install --upgrade pip + pip install -r requirements.txt + make grpc-compile + systemctl daemon-reload + systemctl start $SERVICE_NAME + echo "Update complete" + ;; +uninstall) + check_sudo + read -p "Remove all services and data? (yes/no): " confirm + [ "$confirm" != "yes" ] && echo "Cancelled" && exit 0 + systemctl stop $SERVICE_NAME 2>/dev/null || true + systemctl disable $SERVICE_NAME 2>/dev/null || true + rm -f /etc/systemd/system/relaysms-gateway-server*.{service,target} + systemctl daemon-reload + rm -rf "$INSTALL_DIR" + echo "Uninstall complete" + ;; +*) + echo "Usage: $0 {start|stop|restart|status|logs|enable|disable|update|uninstall}" + exit 1 + ;; +esac diff --git a/mccmnc.py b/mccmnc.py index 5796109..536ad63 100644 --- a/mccmnc.py +++ b/mccmnc.py @@ -1,13 +1,6 @@ """ -A module for matching Mobile Country Codes (MCC) and Mobile Network Codes (MNC) -against a JSON dataset scraped from https://www.mcc-mnc.com/. - -References: - - Original codebase: https://github.com/jbjulia/mcc-mnc - This module is based on the code from the GitHub repository linked above. - It provides functionality to match MCCs and MNCs against a JSON dataset - containing Public Land Mobile Network (PLMN) information scraped from the - MCC-MNC website. +A module for matching Mobile Country Codes (MCC) and Mobile Network Codes (MNC) +against a JSON dataset from https://www.mcc-mnc.com/. """ import json @@ -16,10 +9,9 @@ from urllib.error import URLError from urllib.request import urlopen -from bs4 import BeautifulSoup from tqdm import tqdm -MCC_MNC_URL = "https://www.mcc-mnc.com/" +MCC_MNC_API_URL = "https://mcc-mnc.com/api/v1/mcc-mnc.php" JSON_PATH = os.path.join(os.path.dirname(__file__), "mccmnc.json") @@ -62,15 +54,23 @@ def find_matches( def update(): """ - Update the JSON data by scraping the MCC-MNC website. + Update the JSON data by fetching from the MCC-MNC API. Returns: None """ try: - with urlopen(MCC_MNC_URL) as raw: - print(f"Decoding raw HTML from {MCC_MNC_URL}") - soup = BeautifulSoup(raw, features="html.parser") + print(f"Fetching MCC-MNC data from API: {MCC_MNC_API_URL}") + with urlopen(MCC_MNC_API_URL, timeout=30) as response: + api_data = json.loads(response.read().decode("utf-8")) + + if "data" not in api_data: + print("Error: Invalid API response format") + sys.exit(1) + + entries = api_data["data"] + total_entries = len(entries) + print(f"Received {total_entries} entries from API") if os.path.exists(JSON_PATH): print(f"Removing old JSON dictionary {JSON_PATH}.") @@ -78,29 +78,26 @@ def update(): print(f"Creating new JSON dictionary {JSON_PATH}.") json_data = {} - table = soup.find("table") - rows = table.find_all("tr")[1:] # Skip the header - total_rows = len(rows) + progress_bar = tqdm( - total=total_rows, + total=total_entries, bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt}", colour="blue", ) - for i, row in enumerate(rows, start=1): - cols = row.find_all("td") - mcc = cols[0].text - mnc = cols[1].text + for i, entry in enumerate(entries, start=1): + mcc = entry.get("mcc", "") + mnc = entry.get("mnc", "") plmn = mcc + mnc # MCC + MNC json_data[plmn] = { "MCC": mcc, "MNC": mnc, - "ISO": cols[2].text, - "COUNTRY": cols[3].text, - "CC": cols[4].text, - "NETWORK": cols[5].text.strip() if cols[5].text else "unknown", + "ISO": entry.get("iso", ""), + "COUNTRY": entry.get("country", ""), + "CC": entry.get("countryCode", ""), + "NETWORK": entry.get("network", "unknown"), } - progress_bar.set_description(f"Processing row {i}/{total_rows}") + progress_bar.set_description(f"Processing entry {i}/{total_entries}") progress_bar.update(1) progress_bar.close() @@ -109,6 +106,11 @@ def update(): print(f"\nSaving JSON dictionary to {JSON_PATH}.") json.dump(json_data, json_file, indent=4, sort_keys=True) + print(f"Successfully updated MCC-MNC data with {len(json_data)} entries.") + except URLError as e: - print(f"Error downloading file: {e}") + print(f"Error downloading from API: {e}") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error parsing JSON response: {e}") sys.exit(1) diff --git a/relaysms-gateway-server-ftp.service b/relaysms-gateway-server-ftp.service new file mode 100644 index 0000000..0cc6a50 --- /dev/null +++ b/relaysms-gateway-server-ftp.service @@ -0,0 +1,14 @@ +[Unit] +Description=RelaySMS Gateway Server FTP Server +PartOf=relaysms-gateway-server.target +After=network.target + +[Service] +WorkingDirectory=/opt/relaysms/relaysms-gateway-server +EnvironmentFile=/opt/relaysms/relaysms-gateway-server/.env +ExecStart=/opt/relaysms/relaysms-gateway-server/venv/bin/python3 -u -m src.ftp_server +SyslogIdentifier=relaysms-gateway-server-ftp +ReadWritePaths=/opt/relaysms/relaysms-gateway-server/data + +[Install] +WantedBy=relaysms-gateway-server.target diff --git a/relaysms-gateway-server-imap.service b/relaysms-gateway-server-imap.service new file mode 100644 index 0000000..c8385c3 --- /dev/null +++ b/relaysms-gateway-server-imap.service @@ -0,0 +1,15 @@ +[Unit] +Description=RelaySMS Gateway Server IMAP Listener +PartOf=relaysms-gateway-server.target +After=network.target + +[Service] +WorkingDirectory=/opt/relaysms/relaysms-gateway-server +EnvironmentFile=/opt/relaysms/relaysms-gateway-server/.env +ExecStart=/opt/relaysms/relaysms-gateway-server/venv/bin/python3 -u -m src.imap_listener +Restart=on-failure +SyslogIdentifier=relaysms-gateway-server-imap +ReadWritePaths=/opt/relaysms/relaysms-gateway-server/data + +[Install] +WantedBy=relaysms-gateway-server.target diff --git a/relaysms-gateway-server-rest.service b/relaysms-gateway-server-rest.service new file mode 100644 index 0000000..8735212 --- /dev/null +++ b/relaysms-gateway-server-rest.service @@ -0,0 +1,15 @@ +[Unit] +Description=RelaySMS Gateway Server REST API +PartOf=relaysms-gateway-server.target +After=network.target + +[Service] +WorkingDirectory=/opt/relaysms/relaysms-gateway-server +EnvironmentFile=/opt/relaysms/relaysms-gateway-server/.env +ExecStart=/opt/relaysms/relaysms-gateway-server/venv/bin/python3 -m gunicorn -w 4 -b ${HOST}:${PORT} --log-level=info --access-logfile=- --threads 15 --timeout 30 main:app +Restart=on-failure +SyslogIdentifier=relaysms-gateway-server-rest +ReadWritePaths=/opt/relaysms/relaysms-gateway-server/data + +[Install] +WantedBy=relaysms-gateway-server.target diff --git a/relaysms-gateway-server.target b/relaysms-gateway-server.target new file mode 100644 index 0000000..e97efdc --- /dev/null +++ b/relaysms-gateway-server.target @@ -0,0 +1,7 @@ +[Unit] +Description=RelaySMS Gateway Server +Wants=relaysms-gateway-server-rest.service relaysms-gateway-server-imap.service relaysms-gateway-server-ftp.service +After=relaysms-gateway-server-rest.service relaysms-gateway-server-imap.service relaysms-gateway-server-ftp.service + +[Install] +WantedBy=multi-user.target diff --git a/requirements.txt b/requirements.txt index f6e55b5..35fa67b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,18 +6,18 @@ Flask-Cors==6.0.2 grpcio==1.80.0 grpcio-testing==1.80.0 grpcio-tools==1.80.0 -gunicorn==25.3.0 -idna==3.13 +gunicorn==26.0.0 +idna==3.14 imap-tools==1.12.1 peewee==4.0.5 -phonenumbers==9.0.29 -pika==1.3.2 +phonenumbers==9.0.30 +pika==1.4.0 pycryptodome==3.23.0 pycryptodomex==3.23.0 pyftpdlib==2.2.0 pymysql==1.1.3 pyOpenSSL==26.2.0 -requests==2.33.1 +requests==2.34.0 tqdm==4.67.3 webencodings==0.5.1 websocket==0.2.1 diff --git a/template.env b/template.env index c74b04c..f76a29b 100644 --- a/template.env +++ b/template.env @@ -23,7 +23,7 @@ MYSQL_HOST=127.0.0.1 MYSQL_USER= MYSQL_PASSWORD= MYSQL_DATABASE=relaysms_gateway_server -SQLITE_DATABASE_PATH=gateway_server.db +SQLITE_DATABASE_PATH=data/gateway_server.db # IMAP Configuration IMAP_SERVER= @@ -42,7 +42,7 @@ FTP_READ_LIMIT=51200 FTP_WRITE_LIMIT=51200 FTP_MAX_CON=256 FTP_MAX_CON_PER_IP=5 -FTP_DIRECTORY=ftp_file_store +FTP_DIRECTORY=data/ftp_file_store # Security SMTP_ALLOWED_EMAIL_ADDRESSES=