Merge pull request #210 from static-frame/copilot/buffered-byte-writing

Add C-level buffered array writer with direct file descriptor writes

Author: flexatone

doc/articles/array_to_tuple_array.py

@@ -12,21 +12,24 @@
 
 sys.path.append(os.getcwd())
 
+
 class ArrayProcessor:
     NAME = ''
     SORT = -1
 
     def __init__(self, array: np.ndarray):
         self.array = array
 
-#-------------------------------------------------------------------------------
+
+# -------------------------------------------------------------------------------
 class AKArray2D1D(ArrayProcessor):
     NAME = 'ak.array_to_tuple_array()'
     SORT = 0
 
     def __call__(self):
         _ = array_to_tuple_array(self.array)
 
+
 class PyArray2D1D(ArrayProcessor):
     NAME = 'Python construction'
     SORT = 1
@@ -41,9 +44,11 @@ def __call__(self):
                 post[i] = tuple(row)
         post.flags.writeable = False
 
-#-------------------------------------------------------------------------------
+
+# -------------------------------------------------------------------------------
 NUMBER = 200
 
+
 def seconds_to_display(seconds: float) -> str:
     seconds /= NUMBER
     if seconds < 1e-4:
@@ -67,9 +72,12 @@ def plot_performance(frame):
     # category is the size of the array
     for cat_count, (cat_label, cat) in enumerate(frame.groupby('size')):
         # each fixture is a collection of tests for one display
-        fixtures = {fixture_label: fixture for fixture_label, fixture in cat.groupby('fixture')}
+        fixtures = {
+            fixture_label: fixture for fixture_label, fixture in cat.groupby('fixture')
+        }
         for fixture_count, (fixture_label, fixture) in enumerate(
-                (k, fixtures[k]) for k in FixtureFactory.DENSITY_TO_DISPLAY):
+            (k, fixtures[k]) for k in FixtureFactory.DENSITY_TO_DISPLAY
+        ):
             ax = axes[cat_count][fixture_count]
 
             # set order
@@ -87,37 +95,46 @@ def plot_performance(frame):
             title = f'{cat_label:.0e}\n{FixtureFactory.DENSITY_TO_DISPLAY[fixture_label]}'
 
             ax.set_title(title, fontsize=6)
-            ax.set_box_aspect(0.75) # makes taller than wide
+            ax.set_box_aspect(0.75)  # makes taller than wide
             time_max = fixture['time'].max()
             ax.set_yticks([0, time_max * 0.5, time_max])
-            ax.set_yticklabels(['',
-                    seconds_to_display(time_max * .5),
+            ax.set_yticklabels(
+                [
+                    '',
+                    seconds_to_display(time_max * 0.5),
                     seconds_to_display(time_max),
-                    ], fontsize=4)
+                ],
+                fontsize=4,
+            )
             # ax.set_xticks(x, names_display, rotation='vertical')
             ax.tick_params(
-                    axis='x',
-                    which='both',
-                    bottom=False,
-                    top=False,
-                    labelbottom=False,
-                    )
-
-    fig.set_size_inches(8, 4) # width, height
+                axis='x',
+                which='both',
+                bottom=False,
+                top=False,
+                labelbottom=False,
+            )
+
+    fig.set_size_inches(8, 4)  # width, height
     fig.legend(post, names_display, loc='center right', fontsize=6)
     # horizontal, vertical
-    fig.text(.05, .96, f'array_to_tuple_array() Performance: {NUMBER} Iterations', fontsize=10)
-    fig.text(.05, .90, get_versions(), fontsize=6)
+    fig.text(
+        0.05,
+        0.96,
+        f'array_to_tuple_array() Performance: {NUMBER} Iterations',
+        fontsize=10,
+    )
+    fig.text(0.05, 0.90, get_versions(), fontsize=6)
 
     fp = '/tmp/array_to_tuple_array.png'
     plt.subplots_adjust(
-            left=0.05,
-            bottom=0.05,
-            right=0.8,
-            top=0.85,
-            wspace=1.0, # width
-            hspace=0.5,
-            )
+        left=0.05,
+        bottom=0.05,
+        right=0.8,
+        top=0.85,
+        wspace=1.0,  # width
+        hspace=0.5,
+    )
     # plt.rcParams.update({'font.size': 22})
     plt.savefig(fp, dpi=300)
 
@@ -127,7 +144,8 @@ def plot_performance(frame):
         os.system(f'open {fp}')
 
 
-#-------------------------------------------------------------------------------
+# -------------------------------------------------------------------------------
+
 
 class FixtureFactory:
     NAME = ''
@@ -136,7 +154,7 @@ class FixtureFactory:
     def get_array(size: int, width_ratio: int) -> np.ndarray:
         if width_ratio > 1:
             return np.arange(size).reshape(size // width_ratio, width_ratio)
-        return np.arange(size) # return 1D array
+        return np.arange(size)  # return 1D array
 
     @classmethod
     def get_label_array(cls, size: int) -> tp.Tuple[str, np.ndarray]:
@@ -174,6 +192,7 @@ def get_array(size: int) -> np.ndarray:
         a = FixtureFactory.get_array(size, 2)
         return a
 
+
 class FFC5(FixtureFactory):
     NAME = 'column-5'
 
@@ -182,6 +201,7 @@ def get_array(size: int) -> np.ndarray:
         a = FixtureFactory.get_array(size, 5)
         return a
 
+
 class FFC10(FixtureFactory):
     NAME = 'column-10'
 
@@ -190,6 +210,7 @@ def get_array(size: int) -> np.ndarray:
         a = FixtureFactory.get_array(size, 10)
         return a
 
+
 class FFC20(FixtureFactory):
     NAME = 'column-20'
 
@@ -198,15 +219,17 @@ def get_array(size: int) -> np.ndarray:
         a = FixtureFactory.get_array(size, 20)
         return a
 
+
 def get_versions() -> str:
     import platform
+
     return f'OS: {platform.system()} / ArrayKit: {ak.__version__} / NumPy: {np.__version__}\n'
 
 
 CLS_PROCESSOR = (
     AKArray2D1D,
     PyArray2D1D,
-    )
+)
 
 CLS_FF = (
     FFC1,
@@ -228,26 +251,20 @@ def run_test():
                 record = [cls, NUMBER, fixture_label, size]
                 print(record)
                 try:
-                    result = timeit.timeit(
-                            f'runner()',
-                            globals=locals(),
-                            number=NUMBER)
+                    result = timeit.timeit(f'runner()', globals=locals(), number=NUMBER)
                 except OSError:
                     result = np.nan
                 finally:
                     pass
                 record.append(result)
                 records.append(record)
 
-    f = pd.DataFrame.from_records(records,
-            columns=('cls_processor', 'number', 'fixture', 'size', 'time')
-            )
+    f = pd.DataFrame.from_records(
+        records, columns=('cls_processor', 'number', 'fixture', 'size', 'time')
+    )
     print(f)
     plot_performance(f)
 
-if __name__ == '__main__':
 
+if __name__ == '__main__':
     run_test()
-
-
-