monitor_armobs.py

#!/usr/bin/env python
# coding: utf-8

# # SMNA-Dashboard
#  
# Este notebook trata da apresentação do espaço utilizado pelos arquivos de observações disponíveis para a utilização com o SMNA. As informações apresentadas não representam quantidades ou tipos de dados envolvidos ou utiizados no processo de assimilação de dados, mas apenas o espaço em disco utilizado por estes. As informações mais importantes que podem ser obtidas com este dashboard são o espaço em disco total utilizado por diferentes tipos de observações, separadas por horário sinótico, período e tipo de dados.
#  
# **Nota:** Se o slider que permite ajustar o período a ser visualizado não atualizar a tabela por completo (e.g., o slider está ajustado até a data 13 de Setembro, mas a tabela mostra resultados apenas até o dia 4 de Setembro), pode ser um indicativo de que os arquivos de observação não se encontram no disco verificado. 
# 
# Para realizar o deploy do dashboard no GitHub, é necessário converter este notebook em um script executável, o que pode ser feito a partir da interface do Jupyter (`File` -> `Save and Export Notebook As...` -> `Executable Script`). A seguir, utilize o comando abaixo para converter o script em uma página HTML. Junto com a página, será gerado um arquivo JavaScript e ambos devem ser adicionados ao repositório, junto com o arquivo CSV.
#  
# ```
# panel convert SMNA-Dashboard.py --to pyodide-worker --out .
# ```
# 
# Para utilizar o dashboard localmente, utilize o comando a seguir:
# 
# ```
# panel serve SMNA-Dashboard.ipynb --autoreload --show
#  ```
#  
# ---
# Carlos Frederico Bastarz (carlos.bastarz@inpe.br), Setembro de 2023.

import io
import os
import glob
import pandas as pd
import hvplot.pandas
import holoviews as hv
import panel as pn
#from datetime import timedelta
import requests
from math import pi
from bokeh.palettes import Category20c, Category20
from bokeh.plotting import figure
from bokeh.transform import cumsum
from bokeh.models.widgets.tables import DateFormatter
from datetime import datetime, timedelta

from monitor_texts import MonitoringAppTexts
from monitor_dates import MonitoringAppDates

pn.extension('floatpanel')
pn.extension(sizing_mode="stretch_width", notifications=True)

monitor_app_texts = MonitoringAppTexts()
monitor_warning_bottom_main = monitor_app_texts.warnings()

def url_exists(url):
    try:
        response = requests.head(url, allow_redirects=True, timeout=5)
        # códigos 200-399 indicam que a URL está acessível
        if response.status_code < 400:
            print(f"✅ [OBS STORAGE] Arquivo acessível: {url}")
            dfs_obj = pd.read_csv(url, header=[0], parse_dates=[('Data do Download'), ('Data da Observação')])
            return True, dfs_obj
        else:
            print(f"❌ [OBS STORAGE] Arquivo não encontrado: {url} (status {response.status_code})")
            return False, None
    except requests.RequestException:
        return False, None

dfs_file = 'https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/obsm/mon_rec_obs_final.csv'

dfs_exists = url_exists(dfs_file)

if dfs_exists[0]:

    dfs = url_exists(dfs_file)[1]

    dfs['Data do Download'] = pd.to_datetime(dfs['Data do Download'], errors='coerce')
    dfs['Data da Observação'] = pd.to_datetime(dfs['Data da Observação'], errors='coerce')

    dfs['Diferença de Tempo'] = (dfs['Data do Download'] - dfs['Data da Observação']) - timedelta(hours=3)

    monitoring_app_dates = MonitoringAppDates()
    sdate = monitoring_app_dates.getDates()[0].strip()
    edate = monitoring_app_dates.getDates()[1].strip()

    start_date = datetime(int(sdate[0:4]), int(sdate[4:6]), int(sdate[6:8]), int(sdate[8:10]))
    end_date = datetime(int(edate[0:4]), int(edate[4:6]), int(edate[6:8]), int(edate[8:10]))

    #date_range = [d.strftime('%Y%m%d%H') for d in pd.date_range(start_date, end_date, freq='6h')][::-1]
    #date = pn.widgets.Select(name='Date', value=date_range[-1], options=date_range)

    #start_date = pd.Timestamp('2023-01-01 00:00:00')
    #end_date = pd.Timestamp('2023-09-13 00:00:00')

    #date_range_slider = pn.widgets.DateRangeSlider(
    #    name='Intervalo',
    #    start=start_date, end=end_date,
    #    value=(start_date, end_date),
    #    step=24*3600*1000,
    #    orientation='horizontal'
    #)

    values = (start_date, end_date)
    date_range_slider = pn.widgets.DatetimeRangePicker(name='Date Range', value=values, enable_time=False, width=240)

    units = ['KB', 'MB', 'GB', 'TB', 'PB']
    otype = list(dfs['Tipo de Observação'].unique())
    ftype = ['gdas', 'gfs']
    synoptic_time_list = ['00Z', '06Z', '12Z', '18Z', '00Z e 12Z', '06Z e 18Z', '00Z, 06Z, 12Z e 18Z']

    units_w = pn.widgets.Select(name='Unit', value=units[2], options=units, width=240)
    otype_w = pn.widgets.MultiChoice(name='Observation type', value=otype, options=otype, solid=False, width=260)
    ftype_w = pn.widgets.MultiChoice(name='File type', value=[ftype[0]], options=ftype, solid=False, width=260)
    synoptic_time = pn.widgets.RadioBoxGroup(name='Synopit time', value=synoptic_time_list[-1], options=synoptic_time_list, inline=False, width=240)

    date_range = date_range_slider.value

    dic_size = {}
    def getSizeDic(dfsp, otype_w):
        dfsp_tot_down_otype = dfsp['Tamanho do Download (KB)'].loc[dfsp['Tipo de Observação'] == otype_w[-1]].sum(axis=0)
        dic_size[otype_w[-1]] = dfsp_tot_down_otype
        return dic_size

    def subDataframe(df, start_date, end_date):
        mask = (df['Data da Observação'] >= start_date) & (df['Data da Observação'] <= end_date)
        return df.loc[mask]

    def subTimeDataFrame(synoptic_time):
        if synoptic_time == '00Z': time_fmt0 = '00:00:00'; time_fmt1 = '00:00:00'
        if synoptic_time == '06Z': time_fmt0 = '06:00:00'; time_fmt1 = '06:00:00'
        if synoptic_time == '12Z': time_fmt0 = '12:00:00'; time_fmt1 = '12:00:00'
        if synoptic_time == '18Z': time_fmt0 = '18:00:00'; time_fmt1 = '18:00:00'

        if synoptic_time == '00Z e 12Z': time_fmt0 = '00:00:00'; time_fmt1 = '12:00:00'
        if synoptic_time == '06Z e 18Z': time_fmt0 = '06:00:00'; time_fmt1 = '18:00:00'

        if synoptic_time == '00Z e 06Z': time_fmt0 = '00:00:00'; time_fmt1 = '06:00:00'
        if synoptic_time == '12Z e 18Z': time_fmt0 = '12:00:00'; time_fmt1 = '18:00:00'

        if synoptic_time == '00Z, 06Z, 12Z e 18Z': time_fmt0 = '00:00:00'; time_fmt1 = '18:00:00'

        return time_fmt0, time_fmt1

    def unitConvert(units_w):
        if units_w == 'KB':
            factor = float(1)
            n1factor = 'Tamanho do Download (KB)'
            n2factor = 'Tamanho (KB)'
            n3factor = 'Total Armazenado (KB):'
        elif units_w == 'MB':
            factor = float(1 / (1024 ** 2))
            n1factor = 'Tamanho do Download (MB)'
            n2factor = 'Tamanho (MB)'
            n3factor = 'Total Armazenado (MB):'
        elif units_w == 'GB':
            factor = float(1 / (1024 ** 3))
            n1factor = 'Tamanho do Download (GB)'
            n2factor = 'Tamanho (GB)'
            n3factor = 'Total Armazenado (GB):'
        elif units_w == 'TB':
            factor = float(1 / (1024 ** 4))
            n1factor = 'Tamanho do Download (TB)'
            n2factor = 'Tamanho (TB)'
            n3factor = 'Total Armazenado (TB):'
        elif units_w == 'PB':
            factor = float(1 / (1024 ** 5))
            n1factor = 'Tamanho do Download (PB)'
            n2factor = 'Tamanho (PB)'
            n3factor = 'Total Armazenado (PB):'

        return factor, n1factor, n2factor, n3factor

    @pn.depends(otype_w, ftype_w, synoptic_time, date_range_slider.param.value, units_w)
    def getTotDown(otype_w, ftype_w, synoptic_time, date_range, units_w):
        start_date, end_date = date_range
        dfs_tmp = dfs.copy()
        dfs2 = subDataframe(dfs_tmp, start_date, end_date)

        factor, n1factor, n2factor, n3factor = unitConvert(units_w)

        dfs2[n1factor] = dfs2['Tamanho do Download (KB)'].multiply(factor)

        time_fmt0, time_fmt1 = subTimeDataFrame(synoptic_time)

        if time_fmt0 == time_fmt1:
            dfsp = dfs2.loc[dfs2['Tipo de Observação'].isin(otype_w)].loc[dfs2['Tipo de Arquivo'].isin(ftype_w)].set_index('Data da Observação').at_time(str(time_fmt0)).reset_index()
        else:
            dfsp = dfs2.loc[dfs2['Tipo de Observação'].isin(otype_w)].loc[dfs2['Tipo de Arquivo'].isin(ftype_w)].set_index('Data da Observação').between_time(str(time_fmt0), str(time_fmt1), inclusive='both')

            if synoptic_time == '00Z e 12Z':
                dfsp = dfsp.drop(dfsp.at_time('06:00:00').index).reset_index()
            elif synoptic_time == '06Z e 18Z':
                dfsp = dfsp.drop(dfsp.at_time('12:00:00').index).reset_index()
            elif synoptic_time == '00Z, 06Z, 12Z e 18Z':
                dfsp = dfsp.reset_index()

        dfsp_tot_down = dfsp['Tamanho do Download (KB)'].sum(axis=0)

        tot_down = pn.indicators.Number(name=n3factor, value=dfsp_tot_down * factor, format='{value:.2f}', font_size='16pt', title_size='12pt')

        return pn.Column(tot_down, sizing_mode="stretch_both")

    @pn.depends(otype_w, ftype_w, synoptic_time, date_range_slider.param.value, units_w)
    def getTable(otype_w, ftype_w, synoptic_time, date_range, units_w):
        start_date, end_date = date_range
        dfs_tmp = dfs.copy()
        dfs2 = subDataframe(dfs_tmp, start_date, end_date)

        factor, n1factor, n2factor, n3factor = unitConvert(units_w)

        #dfs2[n1factor] = dfs2['Tamanho do Download (KB)'].multiply(factor)
        #dfs2[n1factor] = dfs2[n1factor].apply(lambda x: x*factor)
        #dfs2.loc[:,'Tamanho do Download (KB)'] *= factor

        time_fmt0, time_fmt1 = subTimeDataFrame(synoptic_time)

        if time_fmt0 == time_fmt1:
            dfsp = dfs2.loc[dfs2['Tipo de Observação'].isin(otype_w)].loc[dfs2['Tipo de Arquivo'].isin(ftype_w)].set_index('Data da Observação').at_time(str(time_fmt0)).reset_index()
        else:
            dfsp = dfs2.loc[dfs2['Tipo de Observação'].isin(otype_w)].loc[dfs2['Tipo de Arquivo'].isin(ftype_w)].set_index('Data da Observação').between_time(str(time_fmt0), str(time_fmt1), inclusive='both')

            if synoptic_time == '00Z e 12Z':
                dfsp = dfsp.drop(dfsp.at_time('06:00:00').index).reset_index()
            elif synoptic_time == '06Z e 18Z':
                dfsp = dfsp.drop(dfsp.at_time('12:00:00').index).reset_index()
            elif synoptic_time == '00Z, 06Z, 12Z e 18Z':
                dfsp = dfsp.reset_index()

        bokeh_formatters = {
            'Diferença de Tempo': DateFormatter(format='%H:%M:%S'),
        }

        # Avançado
        #df_tb = pn.pane.DataFrame(dfsp,
        #                          name='DataFrame',
        #                          height=600,
        #                          bold_rows=True,
        #                          border=15,
        #                          decimal='.',
        #                          index=True,
        #                          show_dimensions=True,
        #                          justify='center',
        #                          sparsify=True,
        #                          sizing_mode='stretch_both',
        #                         )

        # Avançado
        #df_tb = pn.widgets.DataFrame(dfsp,
        #                             name='DataFrame',
        #                             height=600,
        #                             show_index=True,
        #                             frozen_rows=0,
        #                             frozen_columns=2,
        #                             autosize_mode='force_fit',
        #                             fit_columns=True,
        #                             formatters=bokeh_formatters,
        #                             auto_edit=False,
        #                             reorderable=True,
        #                             sortable=True,
        #                             text_align='center',
        #                            )

        stylesheet = """
            .tabulator-cell {
            font-size: 12px;
        }
        """

        # Muito Avançado (e pesado)
        df_tb = pn.widgets.Tabulator(dfsp,
                                    name='DataFrame',
                                    #frozen_rows=[0,1],
                                    #frozen_columns=[2],
                                    #pagination=None,
                                    disabled=True,
                                    selectable='toggle',
                                    #show_index=True,
                                    theme='bootstrap4',
                                    text_align='center',
                                    layout='fit_data', #width=400,
                                    stylesheets=[stylesheet],
                                    formatters=bokeh_formatters,
                                    )

        def get_csv():
            io_buffer = io.BytesIO()
            dfsp.to_csv(io_buffer, index=False)
            io_buffer.seek(0)  # Retorna ao início do buffer
            return io_buffer

        file_download = pn.widgets.FileDownload(
            icon='download',
            callback=get_csv,
            filename='obs_storage.csv',
            #filename=lambda: f"dados_{date_range.value[0].strftime('%Y%m%d')}_{date_range.value[1].strftime('%Y%m%d')}.csv",
            button_type='success',
            width=310
        )

        return pn.Column(pn.Column(df_tb, file_download), height=800, sizing_mode="stretch_width")

    @pn.depends(otype_w, ftype_w, synoptic_time, date_range_slider.param.value, units_w)
    def plotLine(otype_w, ftype_w, synoptic_time, date_range, units_w):
        for count, i in enumerate(otype_w):
            for count2, j in enumerate(ftype_w):
                if count == 0:
                    start_date, end_date = date_range
                    dfs_tmp = dfs.copy()
                    dfs2 = subDataframe(dfs_tmp, start_date, end_date)

                    time_fmt0, time_fmt1 = subTimeDataFrame(synoptic_time)

                    notype = otype_w[count]

                    if time_fmt0 == time_fmt1:
                        dfsp = dfs2.loc[dfs2['Tipo de Observação'] == str(i)].loc[dfs2['Tipo de Arquivo'] == str(j)].set_index('Data da Observação').at_time(str(time_fmt0)).reset_index()
                    else:
                        dfsp = dfs2.loc[dfs2['Tipo de Observação'] == str(i)].loc[dfs2['Tipo de Arquivo'] == str(j)].set_index('Data da Observação').between_time(str(time_fmt0), str(time_fmt1), inclusive='both')

                        if synoptic_time == '00Z e 12Z':
                            dfsp = dfsp.drop(dfsp.at_time('06:00:00').index).reset_index()
                        elif synoptic_time == '06Z e 18Z':
                            dfsp = dfsp.drop(dfsp.at_time('12:00:00').index).reset_index()
                        elif synoptic_time == '00Z, 06Z, 12Z e 18Z':
                            dfsp = dfsp.reset_index()

                    factor, n1factor, n2factor, n3factor = unitConvert(units_w)

                    dfsp[n1factor] = dfsp['Tamanho do Download (KB)'].multiply(factor)

                    df_pl = dfsp.hvplot.line(x='Data da Observação', xlabel='Date', y=n1factor,
                                        ylabel=str(n2factor), label=str(notype), rot=90, grid=True,
                                        line_width=3, height=550, responsive=True)

                    df_pl_s = dfsp.hvplot.scatter(x='Data da Observação', y=n1factor, label=str(notype), persist=True, responsive=True).opts(size=5, marker='o')

                else:

                    start_date, end_date = date_range
                    dfs_tmp = dfs.copy()
                    dfs2 = subDataframe(dfs_tmp, start_date, end_date)

                    time_fmt0, time_fmt1 = subTimeDataFrame(synoptic_time)

                    notype = otype_w[count]

                    if time_fmt0 == time_fmt1:
                        dfsp = dfs2.loc[dfs2['Tipo de Observação'] == str(i)].loc[dfs2['Tipo de Arquivo'] == str(j)].set_index('Data da Observação').at_time(str(time_fmt0)).reset_index()
                    else:
                        dfsp = dfs2.loc[dfs2['Tipo de Observação'] == str(i)].loc[dfs2['Tipo de Arquivo'] == str(j)].set_index('Data da Observação').between_time(str(time_fmt0), str(time_fmt1), inclusive='both')

                        if synoptic_time == '00Z e 12Z':
                            dfsp = dfsp.drop(dfsp.at_time('06:00:00').index).reset_index()
                        elif synoptic_time == '06Z e 18Z':
                            dfsp = dfsp.drop(dfsp.at_time('12:00:00').index).reset_index()
                        elif synoptic_time == '00Z, 06Z, 12Z e 18Z':
                            dfsp = dfsp.reset_index()

                    factor, n1factor, n2factor, n3factor = unitConvert(units_w)

                    dfsp[n1factor] = dfsp['Tamanho do Download (KB)'].multiply(factor)

                    df_pl *= dfsp.hvplot.line(x='Data da Observação', xlabel='Date', y=n1factor,
                                        ylabel=n2factor, label=str(notype), rot=90, grid=True,
                                        line_width=3, height=550, responsive=True)

                    df_pl_s *= dfsp.hvplot.scatter(x='Data da Observação', y=n1factor, label=str(notype), persist=True, responsive=True).opts(size=5, marker='o')

        return pn.Column(df_pl * df_pl_s, sizing_mode='stretch_width')

    @pn.depends(otype_w, ftype_w, synoptic_time, date_range_slider.param.value, units_w)
    def plotSelSize(otype_w, ftype_w, synoptic_time, date_range, units_w):
        start_date, end_date = date_range
        dfs_tmp = dfs.copy()
        dfs2 = subDataframe(dfs_tmp, start_date, end_date)

        time_fmt0, time_fmt1 = subTimeDataFrame(synoptic_time)

        if time_fmt0 == time_fmt1:
            dfsp = dfs2.loc[dfs2['Tipo de Observação'].isin(otype_w)].loc[dfs2['Tipo de Arquivo'].isin(ftype_w)].set_index('Data da Observação').at_time(str(time_fmt0)).reset_index()
        else:
            dfsp = dfs2.loc[dfs2['Tipo de Observação'].isin(otype_w)].loc[dfs2['Tipo de Arquivo'].isin(ftype_w)].set_index('Data da Observação').between_time(str(time_fmt0), str(time_fmt1), inclusive='both')

            if synoptic_time == '00Z e 12Z':
                dfsp = dfsp.drop(dfsp.at_time('06:00:00').index).reset_index()
            elif synoptic_time == '06Z e 18Z':
                dfsp = dfsp.drop(dfsp.at_time('12:00:00').index).reset_index()
            elif synoptic_time == '00Z, 06Z, 12Z e 18Z':
                dfsp = dfsp.reset_index()

        factor, n1factor, n2factor, n3factor = unitConvert(units_w)

        # Tamanho do download (ou do espaço ocupado), de acordo com a seleção da tabela
        dfsp_tot_down = dfsp['Tamanho do Download (KB)'].sum(axis=0)

        dfsp_dic_down = getSizeDic(dfsp, otype_w)

        data = pd.Series(dfsp_dic_down).reset_index(name='Tamanho do Download (KB)').rename(columns={'index':'Tipo de Observação'})

        # Acrescenta uma nova coluna 'Tamanho Relativo' à série data
        data['Tamanho Relativo (%)'] = (data['Tamanho do Download (KB)'] / dfsp_tot_down) * 100

        data['angle'] = (data['Tamanho do Download (KB)'] / data['Tamanho do Download (KB)'].sum()) * (2 * pi)
        #data['color'] = Category20c[len(dfsp_dic_down)]
        #if len(dfsp_dic_down) < 3:
        #    data['color'] = '#ffffff'
        #else:
        #    data['color'] = Category20c[len(dfsp_dic_down)]
        if len(dfsp_dic_down) == 0:
            data['color'] = ''
        elif len(dfsp_dic_down) == 1:
            #data['color'] = 'red'
            data['color'] = Category20c[3][0]
        elif len(dfsp_dic_down) == 2:
            #data['color'] = 'blue'
            data['color'] = Category20c[3][1]
        elif len(dfsp_dic_down) > 2:
            data['color'] = Category20c[len(dfsp_dic_down)]
            #data['color'] = Category20c[20][len(dfsp_dic_down)]

        p = figure(height=550, title='Relative size (%)', #toolbar_location=None, tools="hover",
                tooltips="@{Tipo de Observação}: @{Tamanho Relativo (%)}", x_range=(-0.6, 1.15))

        r = p.wedge(x=0, y=1, radius=0.55,
                    start_angle=cumsum('angle', include_zero=True), end_angle=cumsum('angle'),
                    line_color='white', fill_color='color', legend_field='Tipo de Observação',
                    source=data)

        p.axis.axis_label=None
        p.axis.visible=False
        p.grid.grid_line_color=None

        return pn.Column(pn.pane.Bokeh(p))

    card_parameters = pn.Card(pn.Row(date_range_slider, pn.widgets.TooltipIcon(value='Choose a date range', align='start')),
                            pn.Row(synoptic_time, pn.widgets.TooltipIcon(value='Choose a synoptic time', align='start')),
                            pn.Row(units_w, pn.widgets.TooltipIcon(value='Choose a unit', align='start')),
                            pn.Row(pn.Column(ftype_w, height=120), pn.widgets.TooltipIcon(value='Choose a file type', align='start')),
                            pn.Row(pn.Column(otype_w, height=450), pn.widgets.TooltipIcon(value='Choose one or more observation types', align='start')),
                            title='Parameters', collapsed=False)

    #tabs_contents = pn.Tabs(('PLOTS', pn.Row(plotLine, pn.Row(plotSelSize, width=600))), ('TABLE', getTable))
    tabs_contents = pn.Tabs(('PLOTS', plotLine), ('TABLE', getTable), dynamic=True)
    #tabs_contents = pn.Tabs(('PLOTS', pn.Row(pn.Column('Time series for the observation storage.', plotLine), pn.Column('Pie chart for the observation storage relative to the chosen date range.', plotSelSize))), ('TABLE', pn.Column('Summary table for the observation storage.', getTable)))

    def monitor_armobs_sidebar():
        return card_parameters

    def monitor_armobs_main():
        return pn.Column("""
                        # Observation Storage

                        Set the parameters on the sidebar to update the plots. Click on the `TABLE` tab to get an overview of the observation stored.
                        """, tabs_contents,  monitor_warning_bottom_main, sizing_mode='stretch_width')
else:

    def monitor_armobs_sidebar():
        return pn.Column("""
                        # Observation Storage

                        Set the parameters on the sidebar to update the plots. Click on the `TABLE` tab to get an overview of the observation stored.
                        """, tabs_contents,  monitor_warning_bottom_main, sizing_mode='stretch_width')

    def monitor_armobs_main():
        return pn.Column("""
                        # Observation Storage

                        Set the parameters on the sidebar to update the plots. Click on the `TABLE` tab to get an overview of the observation stored.
                        """, tabs_contents,  monitor_warning_bottom_main, sizing_mode='stretch_width')