|
5 | 5 | <head> |
6 | 6 | <meta charset="utf-8" /> |
7 | 7 | <meta name="viewport" content="width=device-width, initial-scale=1.0" /> |
8 | | - <title>dataretrieval.utils — dataretrieval 0.1.dev1+g18f831f1d documentation</title> |
| 8 | + <title>dataretrieval.utils — dataretrieval 0.1.dev1+g0f90912bc documentation</title> |
9 | 9 | <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" /> |
10 | 10 | <link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" /> |
11 | 11 |
|
12 | 12 |
|
13 | 13 | <script src="../../_static/jquery.js?v=5d32c60e"></script> |
14 | 14 | <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script> |
15 | | - <script src="../../_static/documentation_options.js?v=de132bfb"></script> |
| 15 | + <script src="../../_static/documentation_options.js?v=bc18b2be"></script> |
16 | 16 | <script src="../../_static/doctools.js?v=fd6eb6e6"></script> |
17 | 17 | <script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script> |
18 | 18 | <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script> |
@@ -176,6 +176,114 @@ <h1>Source code for dataretrieval.utils</h1><div class="highlight"><pre> |
176 | 176 |
|
177 | 177 |
|
178 | 178 |
|
| 179 | +<span class="c1"># (time-suffix, tz-suffix) pairs that follow a "<prefix>Date" column.</span> |
| 180 | +<span class="n">_TIME_TZ_SUFFIXES</span> <span class="o">=</span> <span class="p">(</span> |
| 181 | + <span class="c1"># WQX3 / Samples, e.g.</span> |
| 182 | + <span class="c1"># Activity_StartDate / Activity_StartTime / Activity_StartTimeZone</span> |
| 183 | + <span class="p">(</span><span class="s2">"Time"</span><span class="p">,</span> <span class="s2">"TimeZone"</span><span class="p">),</span> |
| 184 | + <span class="c1"># Legacy WQP (slash-separated), e.g.</span> |
| 185 | + <span class="c1"># ActivityStartDate / ActivityStartTime/Time / ActivityStartTime/TimeZoneCode</span> |
| 186 | + <span class="p">(</span><span class="s2">"Time/Time"</span><span class="p">,</span> <span class="s2">"Time/TimeZoneCode"</span><span class="p">),</span> |
| 187 | +<span class="p">)</span> |
| 188 | + |
| 189 | + |
| 190 | +<div class="viewcode-block" id="_build_utc_datetime"> |
| 191 | +<a class="viewcode-back" href="../../reference/utils.html#dataretrieval.utils._build_utc_datetime">[docs]</a> |
| 192 | +<span class="k">def</span><span class="w"> </span><span class="nf">_build_utc_datetime</span><span class="p">(</span> |
| 193 | + <span class="n">date_series</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">,</span> <span class="n">time_series</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">,</span> <span class="n">tz_series</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span> |
| 194 | +<span class="p">)</span> <span class="o">-></span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">:</span> |
| 195 | +<span class="w"> </span><span class="sd">"""Combine date + time + tz-abbreviation columns into a UTC pandas Series.</span> |
| 196 | + |
| 197 | +<span class="sd"> Unknown timezone codes (and rows missing any of the three values) yield</span> |
| 198 | +<span class="sd"> ``NaT``. The input columns are not mutated.</span> |
| 199 | +<span class="sd"> """</span> |
| 200 | + <span class="n">offsets</span> <span class="o">=</span> <span class="n">tz_series</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">tz</span><span class="p">)</span> |
| 201 | + <span class="n">combined</span> <span class="o">=</span> <span class="p">(</span> |
| 202 | + <span class="n">date_series</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">"string"</span><span class="p">)</span> |
| 203 | + <span class="o">+</span> <span class="s2">" "</span> |
| 204 | + <span class="o">+</span> <span class="n">time_series</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">"string"</span><span class="p">)</span> |
| 205 | + <span class="o">+</span> <span class="s2">" "</span> |
| 206 | + <span class="o">+</span> <span class="n">offsets</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">"string"</span><span class="p">)</span> |
| 207 | + <span class="p">)</span> |
| 208 | + <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">to_datetime</span><span class="p">(</span> |
| 209 | + <span class="n">combined</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S %z"</span><span class="p">,</span> <span class="n">utc</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"coerce"</span> |
| 210 | + <span class="p">)</span></div> |
| 211 | + |
| 212 | + |
| 213 | + |
| 214 | +<div class="viewcode-block" id="_attach_datetime_columns"> |
| 215 | +<a class="viewcode-back" href="../../reference/utils.html#dataretrieval.utils._attach_datetime_columns">[docs]</a> |
| 216 | +<span class="k">def</span><span class="w"> </span><span class="nf">_attach_datetime_columns</span><span class="p">(</span><span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-></span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span> |
| 217 | +<span class="w"> </span><span class="sd">"""Add ``<prefix>DateTime`` UTC columns for any Date/Time/TimeZone triplets</span> |
| 218 | +<span class="sd"> and sort the frame by the activity-start datetime.</span> |
| 219 | + |
| 220 | +<span class="sd"> Detects two naming patterns that appear in USGS Samples and Water Quality</span> |
| 221 | +<span class="sd"> Portal CSV responses:</span> |
| 222 | + |
| 223 | +<span class="sd"> * **WQX3** — ``<prefix>Date``, ``<prefix>Time``, ``<prefix>TimeZone``</span> |
| 224 | +<span class="sd"> * **Legacy WQP** — ``<prefix>Date``, ``<prefix>Time/Time``,</span> |
| 225 | +<span class="sd"> ``<prefix>Time/TimeZoneCode``</span> |
| 226 | + |
| 227 | +<span class="sd"> For every triplet present, a new ``<prefix>DateTime`` column is appended</span> |
| 228 | +<span class="sd"> holding a UTC ``Timestamp`` (offsets resolved via</span> |
| 229 | +<span class="sd"> :data:`dataretrieval.codes.tz`). The original Date/Time/TimeZone columns</span> |
| 230 | +<span class="sd"> are left intact, and an existing ``<prefix>DateTime`` column is never</span> |
| 231 | +<span class="sd"> overwritten.</span> |
| 232 | + |
| 233 | +<span class="sd"> Rows are sorted (and the index reset) by the canonical activity-start</span> |
| 234 | +<span class="sd"> datetime when present — ``Activity_StartDateTime`` (WQX3) or</span> |
| 235 | +<span class="sd"> ``ActivityStartDateTime`` (legacy WQP) — falling back to the first</span> |
| 236 | +<span class="sd"> detected ``*Date`` column. Mirrors R ``dataRetrieval``'s</span> |
| 237 | +<span class="sd"> end-of-pipeline sort in ``importWQP.R``.</span> |
| 238 | + |
| 239 | +<span class="sd"> Parameters</span> |
| 240 | +<span class="sd"> ----------</span> |
| 241 | +<span class="sd"> df : ``pandas.DataFrame``</span> |
| 242 | +<span class="sd"> DataFrame returned from a Samples or WQP CSV endpoint.</span> |
| 243 | + |
| 244 | +<span class="sd"> Returns</span> |
| 245 | +<span class="sd"> -------</span> |
| 246 | +<span class="sd"> df : ``pandas.DataFrame``</span> |
| 247 | +<span class="sd"> A new DataFrame with derivable ``<prefix>DateTime`` columns appended</span> |
| 248 | +<span class="sd"> and rows sorted by the activity-start datetime (if any date column</span> |
| 249 | +<span class="sd"> was detected).</span> |
| 250 | +<span class="sd"> """</span> |
| 251 | + <span class="n">columns</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">)</span> |
| 252 | + <span class="n">new_columns</span> <span class="o">=</span> <span class="p">{}</span> |
| 253 | + <span class="n">first_date_col</span> <span class="o">=</span> <span class="kc">None</span> |
| 254 | + <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span> |
| 255 | + <span class="k">if</span> <span class="ow">not</span> <span class="n">col</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">"Date"</span><span class="p">):</span> |
| 256 | + <span class="k">continue</span> |
| 257 | + <span class="k">if</span> <span class="n">first_date_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> |
| 258 | + <span class="n">first_date_col</span> <span class="o">=</span> <span class="n">col</span> |
| 259 | + <span class="n">prefix</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">removesuffix</span><span class="p">(</span><span class="s2">"Date"</span><span class="p">)</span> |
| 260 | + <span class="n">target</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="s2">"DateTime"</span> |
| 261 | + <span class="k">if</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">columns</span> <span class="ow">or</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">new_columns</span><span class="p">:</span> |
| 262 | + <span class="k">continue</span> |
| 263 | + <span class="k">for</span> <span class="n">time_suffix</span><span class="p">,</span> <span class="n">tz_suffix</span> <span class="ow">in</span> <span class="n">_TIME_TZ_SUFFIXES</span><span class="p">:</span> |
| 264 | + <span class="n">time_col</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">time_suffix</span> |
| 265 | + <span class="n">tz_col</span> <span class="o">=</span> <span class="n">prefix</span> <span class="o">+</span> <span class="n">tz_suffix</span> |
| 266 | + <span class="k">if</span> <span class="n">time_col</span> <span class="ow">in</span> <span class="n">columns</span> <span class="ow">and</span> <span class="n">tz_col</span> <span class="ow">in</span> <span class="n">columns</span><span class="p">:</span> |
| 267 | + <span class="n">new_columns</span><span class="p">[</span><span class="n">target</span><span class="p">]</span> <span class="o">=</span> <span class="n">_build_utc_datetime</span><span class="p">(</span> |
| 268 | + <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">],</span> <span class="n">df</span><span class="p">[</span><span class="n">time_col</span><span class="p">],</span> <span class="n">df</span><span class="p">[</span><span class="n">tz_col</span><span class="p">]</span> |
| 269 | + <span class="p">)</span> |
| 270 | + <span class="k">break</span> |
| 271 | + <span class="k">if</span> <span class="n">new_columns</span><span class="p">:</span> |
| 272 | + <span class="c1"># Concat in one shot — per-column assignment on a wide CSV-derived</span> |
| 273 | + <span class="c1"># frame triggers pandas' fragmentation PerformanceWarning.</span> |
| 274 | + <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">df</span><span class="p">,</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">new_columns</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">index</span><span class="p">)],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span> |
| 275 | + <span class="k">if</span> <span class="s2">"Activity_StartDateTime"</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span> |
| 276 | + <span class="n">sort_key</span> <span class="o">=</span> <span class="s2">"Activity_StartDateTime"</span> |
| 277 | + <span class="k">elif</span> <span class="s2">"ActivityStartDateTime"</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span> |
| 278 | + <span class="n">sort_key</span> <span class="o">=</span> <span class="s2">"ActivityStartDateTime"</span> |
| 279 | + <span class="k">else</span><span class="p">:</span> |
| 280 | + <span class="n">sort_key</span> <span class="o">=</span> <span class="n">first_date_col</span> |
| 281 | + <span class="k">if</span> <span class="n">sort_key</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> |
| 282 | + <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span><span class="n">by</span><span class="o">=</span><span class="n">sort_key</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> |
| 283 | + <span class="k">return</span> <span class="n">df</span></div> |
| 284 | + |
| 285 | + |
| 286 | + |
179 | 287 | <div class="viewcode-block" id="BaseMetadata"> |
180 | 288 | <a class="viewcode-back" href="../../reference/utils.html#dataretrieval.utils.BaseMetadata">[docs]</a> |
181 | 289 | <span class="k">class</span><span class="w"> </span><span class="nc">BaseMetadata</span><span class="p">:</span> |
|
0 commit comments