|
5 | 5 | <head> |
6 | 6 | <meta charset="utf-8" /> |
7 | 7 | <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
8 | | - <title>dataretrieval.nwis — dataretrieval 0.1.dev1+gba11d6fea documentation</title> |
| 8 | + <title>dataretrieval.nwis — dataretrieval 0.1.dev1+g0d079caff documentation</title> |
9 | 9 | <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" /> |
10 | 10 | <link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" /> |
11 | 11 |
|
12 | 12 |
|
13 | 13 | <script src="../../_static/jquery.js?v=5d32c60e"></script> |
14 | 14 | <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script> |
15 | | - <script src="../../_static/documentation_options.js?v=b11accc6"></script> |
| 15 | + <script src="../../_static/documentation_options.js?v=ee6e1908"></script> |
16 | 16 | <script src="../../_static/doctools.js?v=fd6eb6e6"></script> |
17 | 17 | <script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script> |
18 | 18 | <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script> |
@@ -83,12 +83,12 @@ <h1>Source code for dataretrieval.nwis</h1><div class="highlight"><pre> |
83 | 83 | <span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span> |
84 | 84 |
|
85 | 85 | <span class="kn">import</span><span class="w"> </span><span class="nn">warnings</span> |
| 86 | +<span class="kn">from</span><span class="w"> </span><span class="nn">io</span><span class="w"> </span><span class="kn">import</span> <span class="n">StringIO</span> |
86 | 87 | <span class="kn">from</span><span class="w"> </span><span class="nn">json</span><span class="w"> </span><span class="kn">import</span> <span class="n">JSONDecodeError</span> |
87 | 88 |
|
88 | 89 | <span class="kn">import</span><span class="w"> </span><span class="nn">pandas</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pd</span> |
89 | 90 | <span class="kn">import</span><span class="w"> </span><span class="nn">requests</span> |
90 | 91 |
|
91 | | -<span class="kn">from</span><span class="w"> </span><span class="nn">dataretrieval.rdb</span><span class="w"> </span><span class="kn">import</span> <span class="n">read_rdb</span> |
92 | 92 | <span class="kn">from</span><span class="w"> </span><span class="nn">dataretrieval.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">BaseMetadata</span> |
93 | 93 |
|
94 | 94 | <span class="kn">from</span><span class="w"> </span><span class="nn">.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">query</span> |
@@ -120,14 +120,6 @@ <h1>Source code for dataretrieval.nwis</h1><div class="highlight"><pre> |
120 | 120 | <span class="c1"># NAD83</span> |
121 | 121 | <span class="n">_CRS</span> <span class="o">=</span> <span class="s2">"EPSG:4269"</span> |
122 | 122 |
|
123 | | -<span class="n">_NWIS_RDB_DTYPES</span> <span class="o">=</span> <span class="p">{</span> |
124 | | - <span class="s2">"site_no"</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
125 | | - <span class="s2">"dec_long_va"</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span> |
126 | | - <span class="s2">"dec_lat_va"</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span> |
127 | | - <span class="s2">"parm_cd"</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
128 | | - <span class="s2">"parameter_cd"</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
129 | | -<span class="p">}</span> |
130 | | - |
131 | 123 |
|
132 | 124 | <div class="viewcode-block" id="_parse_json_or_raise"> |
133 | 125 | <a class="viewcode-back" href="../../reference/nwis.html#dataretrieval.nwis._parse_json_or_raise">[docs]</a> |
@@ -1161,13 +1153,64 @@ <h1>Source code for dataretrieval.nwis</h1><div class="highlight"><pre> |
1161 | 1153 | <div class="viewcode-block" id="_read_rdb"> |
1162 | 1154 | <a class="viewcode-back" href="../../reference/nwis.html#dataretrieval.nwis._read_rdb">[docs]</a> |
1163 | 1155 | <span class="k">def</span><span class="w"> </span><span class="nf">_read_rdb</span><span class="p">(</span><span class="n">rdb</span><span class="p">):</span> |
1164 | | -<span class="w"> </span><span class="sd">"""Parse an NWIS RDB response and apply NWIS-specific post-processing.</span> |
| 1156 | +<span class="w"> </span><span class="sd">"""</span> |
| 1157 | +<span class="sd"> Convert NWIS rdb table into a ``pandas.dataframe``.</span> |
| 1158 | + |
| 1159 | +<span class="sd"> Parameters</span> |
| 1160 | +<span class="sd"> ----------</span> |
| 1161 | +<span class="sd"> rdb: string</span> |
| 1162 | +<span class="sd"> A string representation of an rdb table</span> |
| 1163 | + |
| 1164 | +<span class="sd"> Returns</span> |
| 1165 | +<span class="sd"> -------</span> |
| 1166 | +<span class="sd"> df: ``pandas.dataframe``</span> |
| 1167 | +<span class="sd"> A formatted pandas data frame</span> |
1165 | 1168 |
|
1166 | | -<span class="sd"> Thin wrapper around :func:`dataretrieval.rdb.read_rdb` that adds the</span> |
1167 | | -<span class="sd"> NWIS column-dtype hints and runs :func:`format_response` (datetime</span> |
1168 | | -<span class="sd"> index, multi-site MultiIndex, optional GeoDataFrame).</span> |
1169 | 1169 | <span class="sd"> """</span> |
1170 | | - <span class="k">return</span> <span class="n">format_response</span><span class="p">(</span><span class="n">read_rdb</span><span class="p">(</span><span class="n">rdb</span><span class="p">,</span> <span class="n">dtypes</span><span class="o">=</span><span class="n">_NWIS_RDB_DTYPES</span><span class="p">))</span></div> |
| 1170 | + <span class="k">if</span> <span class="s2">"<html>"</span> <span class="ow">in</span> <span class="n">rdb</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="ow">or</span> <span class="s2">"<!doctype html>"</span> <span class="ow">in</span> <span class="n">rdb</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span> |
| 1171 | + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> |
| 1172 | + <span class="s2">"Received HTML response instead of RDB. This often indicates "</span> |
| 1173 | + <span class="s2">"that the service has been moved or is currently unavailable."</span> |
| 1174 | + <span class="p">)</span> |
| 1175 | + |
| 1176 | + <span class="n">count</span> <span class="o">=</span> <span class="mi">0</span> |
| 1177 | + <span class="n">lines</span> <span class="o">=</span> <span class="n">rdb</span><span class="o">.</span><span class="n">splitlines</span><span class="p">()</span> |
| 1178 | + |
| 1179 | + <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">lines</span><span class="p">:</span> |
| 1180 | + <span class="c1"># ignore comment lines</span> |
| 1181 | + <span class="k">if</span> <span class="n">line</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"#"</span><span class="p">):</span> |
| 1182 | + <span class="n">count</span> <span class="o">=</span> <span class="n">count</span> <span class="o">+</span> <span class="mi">1</span> |
| 1183 | + |
| 1184 | + <span class="k">else</span><span class="p">:</span> |
| 1185 | + <span class="k">break</span> |
| 1186 | + |
| 1187 | + <span class="k">if</span> <span class="n">count</span> <span class="o">>=</span> <span class="nb">len</span><span class="p">(</span><span class="n">lines</span><span class="p">):</span> |
| 1188 | + <span class="c1"># All lines are comments — the service returned no data rows (e.g.</span> |
| 1189 | + <span class="c1"># "No sites found matching all criteria"). This is a legitimate empty</span> |
| 1190 | + <span class="c1"># result, so return an empty DataFrame rather than raising.</span> |
| 1191 | + <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span> |
| 1192 | + |
| 1193 | + <span class="n">fields</span> <span class="o">=</span> <span class="n">lines</span><span class="p">[</span><span class="n">count</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">"</span><span class="se">\t</span><span class="s2">"</span><span class="p">)</span> |
| 1194 | + <span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="n">field</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">","</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">fields</span> <span class="k">if</span> <span class="n">field</span><span class="o">.</span><span class="n">strip</span><span class="p">()]</span> |
| 1195 | + <span class="n">dtypes</span> <span class="o">=</span> <span class="p">{</span> |
| 1196 | + <span class="s2">"site_no"</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| 1197 | + <span class="s2">"dec_long_va"</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span> |
| 1198 | + <span class="s2">"dec_lat_va"</span><span class="p">:</span> <span class="nb">float</span><span class="p">,</span> |
| 1199 | + <span class="s2">"parm_cd"</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| 1200 | + <span class="s2">"parameter_cd"</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> |
| 1201 | + <span class="p">}</span> |
| 1202 | + |
| 1203 | + <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span> |
| 1204 | + <span class="n">StringIO</span><span class="p">(</span><span class="n">rdb</span><span class="p">),</span> |
| 1205 | + <span class="n">delimiter</span><span class="o">=</span><span class="s2">"</span><span class="se">\t</span><span class="s2">"</span><span class="p">,</span> |
| 1206 | + <span class="n">skiprows</span><span class="o">=</span><span class="n">count</span> <span class="o">+</span> <span class="mi">2</span><span class="p">,</span> |
| 1207 | + <span class="n">names</span><span class="o">=</span><span class="n">fields</span><span class="p">,</span> |
| 1208 | + <span class="n">na_values</span><span class="o">=</span><span class="s2">"NaN"</span><span class="p">,</span> |
| 1209 | + <span class="n">dtype</span><span class="o">=</span><span class="n">dtypes</span><span class="p">,</span> |
| 1210 | + <span class="p">)</span> |
| 1211 | + |
| 1212 | + <span class="n">df</span> <span class="o">=</span> <span class="n">format_response</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> |
| 1213 | + <span class="k">return</span> <span class="n">df</span></div> |
1171 | 1214 |
|
1172 | 1215 |
|
1173 | 1216 |
|
|
0 commit comments