added option --stdin-json-header

Author: gpoore

CHANGELOG.md

@@ -3,6 +3,15 @@
 
 ## v0.9.0 (dev)
 
+* Added option `--stdin-json-header`, which treats the first line of stdin as
+  a header in JSON format containing data about stdin such as file origin.
+  This allows Codebraid Preview to transmit multiple, concatenated files via
+  stdin while still getting correct file names and line numbers in error and
+  warning messages, rather than a file name like `<string>` and a number based
+  on concatenated file length.  It also allows Codebraid Preview and normal
+  Codebraid processing to share the same cache, since Codebraid can associate
+  the cache with specific file paths rather than just generic stdin.
+
 * Fixed a caching bug.  Errors due to missing executables (built-in code
   execution system), missing Jupyter kernels, ambiguous Jupyter kernel names,
   and some Jupyter configuration issues were only reported during an initial

codebraid/cmdline.py

@@ -13,9 +13,11 @@
 
 import argparse
 import io
+import json
 import pathlib
 import sys
 from . import converters
+from . import util
 from .version import __version__ as version
 
 
@@ -88,6 +90,8 @@ def print_help(self):
                                     'Output from execution is written as soon as it becomes available. '
                                     'No document is created. '
                                     'Options --to and --output are only used (if at all) to inform code output formatting.')
+    parser_pandoc.add_argument('--stdin-json-header', action='store_true',
+                               help='Treat the first line of stdin as a header in JSON format containing data about stdin such as file origin.')
     parser_pandoc.add_argument('files', nargs='*', metavar='FILE',
                                help="Files (multiple files are allowed for formats supported by Pandoc)")
     for opts_or_long_opt, narg in PANDOC_OPTIONS.items():
@@ -154,13 +158,43 @@ def pandoc(args):
 
     if not args.files or (len(args.files) == 1 and args.files[0] == '-'):
         paths = None
+        string_origins = None
         try:
             strings = sys.stdin.read()
         except UnicodeDecodeError as e:
-            sys.exit('Input must be UTF-8:\n{0}'.format(e))
+            sys.exit(f'Input must be UTF-8:\n{e}')
+        if args.stdin_json_header:
+            stdin_lines = util.splitlines_lf(strings)
+            if not stdin_lines:
+                sys.exit('Missing data for --stdin-json-header')
+            if len(stdin_lines) == 1:
+                stdin_lines.append('')
+            json_header_str = stdin_lines[0]
+            try:
+                json_header = json.loads(json_header_str)
+            except Exception as e:
+                sys.exit(f'Invalid data for --stdin-json-header:\n{e}')
+            if not isinstance(json_header, dict):
+                sys.exit(f'Invalid data for --stdin-json-header: expected dict')
+            json_header_origins = json_header.get('origins')
+            if json_header_origins is not None:
+                if not isinstance(json_header_origins, list):
+                    sys.exit('Invalid data for --stdin-json-header field "string_origins"')
+                start_line = 1
+                strings = []
+                string_origins = []
+                for entry in json_header_origins:
+                    origin_name = entry['path']
+                    origin_length = entry['lines']
+                    string_origins.append(origin_name)
+                    strings.append('\n'.join(stdin_lines[start_line:start_line+origin_length]))
+                    start_line += origin_length
+            else:
+                strings = '\n'.join(stdin_lines[1:])
     else:
         paths = args.files
         strings = None
+        string_origins = None
 
     code_defaults = {}
     session_defaults = {}
@@ -187,6 +221,7 @@ def pandoc(args):
     with converters.PandocConverter(
         paths=paths,
         strings=strings,
+        string_origins=string_origins,
         from_format=args.from_format,
         pandoc_file_scope=args.pandoc_file_scope,
         no_cache=args.no_cache,

codebraid/converters/base.py

@@ -58,6 +58,7 @@ class Converter(object):
 
     def __init__(self, *,
                  strings: Optional[Union[str, Sequence[str]]]=None,
+                 string_origins: Optional[Union[str, pathlib.Path, Sequence[str], Sequence[pathlib.Path]]]=None,
                  paths: Optional[Union[str, Sequence[str], pathlib.Path, Sequence[pathlib.Path]]]=None,
                  no_cache: Optional[bool]=False,
                  cache_path: Optional[Union[str, pathlib.Path]]=None,
@@ -80,6 +81,8 @@ def __init__(self, *,
         self.session_defaults = session_defaults
 
         if paths is not None and strings is None:
+            if string_origins is not None:
+                raise TypeError
             if isinstance(paths, str):
                 paths = [pathlib.Path(paths)]
             elif isinstance(paths, pathlib.Path):
@@ -135,15 +138,47 @@ def __init__(self, *,
             elif not (isinstance(strings, collections.abc.Sequence) and
                       strings and all(isinstance(x, str) for x in strings)):
                 raise TypeError
+            string_origins_normalized: Optional[Union[Sequence[str], Sequence[pathlib.Path]]]
+            if string_origins is None:
+                string_origins_normalized = None
+            elif isinstance(string_origins, str) or isinstance(string_origins, pathlib.Path):
+                string_origins_normalized = [string_origins]
+            elif (isinstance(string_origins, collections.abc.Sequence) and string_origins and
+                    (all(isinstance(x, str) for x in string_origins) or all(isinstance(x, pathlib.Path) for x in string_origins))):
+                string_origins_normalized = string_origins
+            else:
+                raise TypeError
+
             # Normalize newlines, as if read from file with universal newlines
-            origin_strings = [io.StringIO(s, newline=None).read() or '\n' for s in strings]
-            if len(strings) == 1:
-                origin_names = ['<string>']
+            origin_strings = []
+            for s in strings:
+                if '\r' in s:
+                    origin_strings.append(io.StringIO(s, newline=None).read() or '\n')
+                else:
+                    origin_strings.append(s or '\n')
+            if string_origins_normalized is not None:
+                origin_names = []
+                self.raw_origin_paths = []
+                self.expanded_origin_paths = {}
+                cwd_path = pathlib.Path('.').absolute()
+                for string_origin in string_origins_normalized:
+                    string_origin_path = pathlib.Path(string_origin)
+                    try:
+                        string_origin_rel_path = string_origin_path.relative_to(cwd_path)
+                    except ValueError:
+                        string_origin_rel_path = string_origin_path
+                    origin_name = string_origin_rel_path.as_posix()
+                    origin_names.append(origin_name)
+                    self.raw_origin_paths.append(string_origin_path)
+                    self.expanded_origin_paths[origin_name] = string_origin_path
             else:
-                origin_names = ['<string({0})>'.format(n+1) for n in range(len(strings))]
+                if len(strings) == 1:
+                    origin_names = ['<string>']
+                else:
+                    origin_names = ['<string({0})>'.format(n+1) for n in range(len(strings))]
+                self.raw_origin_paths = None
+                self.expanded_origin_paths = None
             self.origins = collections.OrderedDict(zip(origin_names, origin_strings))
-            self.raw_origin_paths = None
-            self.expanded_origin_paths = None
             if from_format is None:
                 raise TypeError('Document format is required')
             if self.from_formats is not None and from_format not in self.from_formats:

codebraid/version.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 
 from .fmtversion import get_version_plus_info
-__version__, __version_info__ = get_version_plus_info(0, 9, 0, 'dev', 5)
+__version__, __version_info__ = get_version_plus_info(0, 9, 0, 'dev', 6)