mostly complete implementation of --only-code-output

Author: gpoore

codebraid/code_chunks.py

@@ -298,6 +298,7 @@ def finalize_after_copy(self):
         code_chunk = self.code_chunk
         custom_options = self.custom_options
         if self['lang'] is None:
+            self['inherited_lang'] = True
             self['lang'] = code_chunk.copy_chunks[0].options['lang']
         if code_chunk.inline:
             if code_chunk.command == 'paste' and 'show' not in custom_options:
@@ -335,6 +336,7 @@ def finalize_after_copy(self):
     _default_inline_options = {'complete': True,
                                'example': False,
                                'lang': None,
+                               'inherited_lang': False,
                                'outside_main': False}
     _default_block_options = _default_inline_options.copy()
     _default_block_options.update({'code_first_number': 'next',
@@ -814,9 +816,9 @@ def __init__(self,
         if command == 'paste':
             if 'copy' not in custom_options:
                 self.errors.append(message.SourceError('Command "paste" cannot be used without specifying a target via "copy"'))
-            self.has_output = False
+            self.needs_to_copy = True
         else:
-            self.has_output = True  # Whether need output from copying
+            self.needs_to_copy = False
         if 'copy' in self.options:
             self.copy_chunks = []
 
@@ -878,6 +880,18 @@ def code_str(self):
         return code
 
 
+    @property
+    def attr_hash(self):
+        raise NotImplementedError
+
+    @property
+    def code_hash(self):
+        raise NotImplementedError
+
+    def only_code_output(self, format):
+        raise NotImplementedError
+
+
     def finalize_after_copy(self):
         '''
         Finalize options.  This can be redefined by subclasses so that they
@@ -926,7 +940,7 @@ def copy_code(self):
         if self.command == 'paste':
             if all(cc.command == 'code' for cc in copy_chunks):
                 # When possible, simplify the copying resolution process
-                self.has_output = True
+                self.needs_to_copy = False
         self.code_start_line_number = copy_chunks[0].code_start_line_number
 
 
@@ -983,8 +997,16 @@ def copy_output(self):
             self.expr_lines = copy_chunks[0].expr_lines
         self.stdout_start_line_number = copy_chunks[0].stdout_start_line_number
         self.stderr_start_line_number = copy_chunks[0].stderr_start_line_number
-        self.has_output = True
+        self.needs_to_copy = False
+
 
+    @property
+    def as_markup_lines(self):
+        raise NotImplementedError
+
+    @property
+    def as_example_markup_lines(self):
+        raise NotImplementedError
 
     def layout_output(self, output_type, output_format, lines=None):
         '''

codebraid/code_collections.py

@@ -296,7 +296,7 @@ def finalize(self):
             else:
                 msg = 'Final code chunk cannot have "complete" value "false"'
                 last_cc.errors.append(message.SourceError(msg))
-        if self.status.has_errors:
+        if self.status.prevent_exec:
             # Hashes and line numbers are only needed if code will indeed be
             # executed.  It is impossible to determine these in the case of
             # errors like copy errors which leave code for one or more chunks

codebraid/codeprocessors/base.py

@@ -48,6 +48,7 @@ def __init__(self,
                  origin_paths_for_cache: Optional[List[pathlib.Path]],
                  code_defaults: Optional[Dict],
                  session_defaults: Optional[Dict],
+                 only_code_output: bool,
                  progress: Progress):
         self.code_chunks = code_chunks
         self.cross_origin_sessions = cross_origin_sessions
@@ -62,6 +63,7 @@ def __init__(self,
             self._origin_paths_for_cache_as_strings = [p.as_posix() for p in origin_paths_for_cache]
         self.code_defaults = code_defaults
         self.session_defaults = session_defaults
+        self._only_code_output = only_code_output
         self._progress = progress
 
         self._old_cache_index: Optional[Dict] = None
@@ -90,7 +92,7 @@ def exit_code(self) -> int:
         if any(s.status.prevent_exec for s in self._sessions.values()):
             code ^= 0b00000100
         if (any(s.status.has_errors and not s.status.prevent_exec for s in self._sessions.values()) or
-                any(s.status.has_errors and not s.status.prevent_exec for s in self._sources.values())):
+                any(s.status.has_errors for s in self._sources.values())):
             code ^= 0b00001000
         if any(s.status.has_warnings for s in self._sessions.values()):
             code ^= 0b00010000
@@ -110,6 +112,9 @@ def process(self):
         self._resolve_code_copying()
         self._create_sessions_and_sources()
         self._prep_cache()
+        if self._only_code_output and self._cached_sessions:
+            # Resolve output copying as early as possible
+            self._resolve_output_copying(from_cache=True)
 
 
     def exec(self):
@@ -181,6 +186,7 @@ def _set_chunk_keys(self):
         '''
         Assign code chunk session/source keys.
         '''
+        placeholder_lang_num = 0
         for cc in self.code_chunks:
             if cc.execute:
                 if self.cross_origin_sessions:
@@ -193,6 +199,9 @@ def _set_chunk_keys(self):
                 else:
                     key = CodeKey(cc.options['lang'], cc.options['source'], 'source', cc.origin_name)
             cc.key = key
+            if cc.options['inherited_lang']:
+                cc.options['placeholder_lang'] = f'{placeholder_lang_num}_placeholder'
+                placeholder_lang_num += 1
 
 
     def _index_named_code_chunks(self):
@@ -292,7 +301,7 @@ def _resolve_code_copying(self):
             unresolved_chunks = still_unresolved_chunks
             still_unresolved_chunks = []
 
-    def _resolve_output_copying(self):
+    def _resolve_output_copying(self, *, from_cache: bool=False):
         '''
         For code chunks with copying, handle the output copying.  The output
         copying for commands like "paste" must be handled after code is
@@ -303,11 +312,11 @@ def _resolve_output_copying(self):
         # dependencies; that's already been handled in
         # `_resolve_code_copying()`.  Still need to check for errors that
         # prevent copying, since there can be runtime source errors or other
-        # errors related to output.  Code chunks with `.has_output == True`
-        # are copying code-only code chunks and thus have no output for
-        # copying.
+        # errors related to output.  Code chunks with
+        # `.needs_to_copy == False` are copying code-only code chunks and thus
+        # have no output for copying.
         unresolved_chunks = [cc for cc in self.code_chunks
-                             if cc.command == 'paste' and not cc.errors.prevent_exec and not cc.has_output]
+                             if cc.command == 'paste' and not cc.errors.prevent_exec and cc.needs_to_copy]
         still_unresolved_chunks = []
         while True:
             for cc in unresolved_chunks:
@@ -322,11 +331,13 @@ def _resolve_output_copying(self):
                     traceback = '\n'.join(traceback_list)
                     msg = f'Code chunk(s) have error(s) that prevent copying:\n{traceback}'
                     cc.errors.append(message.SourceError(msg))
-                elif any(not copied_cc.has_output for copied_cc in cc.copy_chunks):
+                    self._progress.source_chunk_complete(cc.source, chunk=cc)
+                elif any(copied_cc.needs_to_copy for copied_cc in cc.copy_chunks):
                     still_unresolved_chunks.append(cc)
                 else:
                     cc.copy_output()
-            if not still_unresolved_chunks:
+                    self._progress.source_chunk_complete(cc.source, chunk=cc)
+            if not still_unresolved_chunks or (from_cache and len(unresolved_chunks) == len(still_unresolved_chunks)):
                 break
             unresolved_chunks = still_unresolved_chunks
             still_unresolved_chunks = []
@@ -347,6 +358,9 @@ def _create_sessions_and_sources(self):
                 self._session_hash_root_sets[session.hash_root].add(session)
         for source in self._sources.values():
             source.finalize()
+            for cc in source.code_chunks:
+                if not cc.needs_to_copy or cc.errors.prevent_exec:
+                    self._progress.source_chunk_complete(cc.source, chunk=cc)
         self._progress.register_sessions(self._sessions.values())
         self._progress.register_sources(self._sources.values())
 

codebraid/codeprocessors/exec_builtin.py

@@ -256,7 +256,7 @@ def _process_code_chunk_output(self, output: bytes, *, code_chunk: CodeChunk, ou
         output_str = self._decode(output, output_type=output_type, code_chunk=code_chunk)
         if output_type == 'stderr':
             output_str = self._sync_stderr_or_compile_output(output_str, code_chunk=code_chunk)
-        getattr(self.progress, f'chunk_{output_type}')(self.session, chunk=code_chunk, output=output_str)
+        getattr(self.progress, f'session_chunk_{output_type}')(self.session, chunk=code_chunk, output=output_str)
         getattr(code_chunk, f'{output_type}_lines').extend(util.splitlines_lf(output_str))
 
     def _run_line_number_to_origin(self, run_line_number: int) -> CodeLineOrigin | tuple[None, None]:
@@ -395,7 +395,7 @@ def process_match(match: re.Match):
     async def _sync_chunk_start_delims(self, code_chunk: CodeChunk):
         self._sync_chunk_start_delims_state[code_chunk.index] += 1
         if self._sync_chunk_start_delims_state[code_chunk.index] == 2:
-            self.progress.chunk_start(self.session, chunk=code_chunk)
+            self.progress.session_chunk_start(self.session, chunk=code_chunk)
             return
         self._sync_chunk_start_delims_waiting += 1
         while self._sync_chunk_start_delims_state[code_chunk.index] < 2:
@@ -413,7 +413,7 @@ async def _sync_chunk_start_delims(self, code_chunk: CodeChunk):
     async def _sync_chunk_end_delims(self, code_chunk: CodeChunk):
         self._sync_chunk_end_delims_state[code_chunk.index] += 1
         if self._sync_chunk_end_delims_state[code_chunk.index] == 2:
-            self.progress.chunk_end(self.session, chunk=code_chunk)
+            self.progress.session_chunk_end(self.session, chunk=code_chunk)
             return
         self._sync_chunk_end_delims_waiting += 1
         while self._sync_chunk_end_delims_state[code_chunk.index] < 2:

codebraid/codeprocessors/exec_jupyter.py

@@ -135,11 +135,11 @@ async def exec(session: Session, *, cache_key_path: pathlib.Path, progress: Prog
                 incomplete_cc_stack.append(cc)
                 continue
             if not incomplete_cc_stack:
-                progress.chunk_start(session, chunk=cc)
+                progress.session_chunk_start(session, chunk=cc)
                 cc_jupyter_id = kernel_client.execute(cc.code_str)
             else:
                 incomplete_cc_stack.append(cc)
-                progress.chunk_start(session, chunk=incomplete_cc_stack[0])
+                progress.session_chunk_start(session, chunk=incomplete_cc_stack[0])
                 cc_jupyter_id = kernel_client.execute('\n'.join(icc.code_str for icc in incomplete_cc_stack))
             deadline = time.monotonic() + session.jupyter_timeout
             while True:
@@ -178,16 +178,16 @@ async def exec(session: Session, *, cache_key_path: pathlib.Path, progress: Prog
                     cc.rich_output.append(rich_output)
                     rich_output_text = kernel_msg_content['data'].get('text/plain')
                     if rich_output_text:
-                        progress.chunk_rich_output_text(session, chunk=cc, output=rich_output_text)
+                        progress.session_chunk_rich_output_text(session, chunk=cc, output=rich_output_text)
                     if rich_output_files:
-                        progress.chunk_rich_output_files(session, chunk=cc, files=rich_output_files.values())
+                        progress.session_chunk_rich_output_files(session, chunk=cc, files=rich_output_files.values())
                 elif kernel_msg_type == 'stream':
                     if kernel_msg_content['name'] == 'stdout':
                         cc.stdout_lines.extend(util.splitlines_lf(kernel_msg_content['text']))
-                        progress.chunk_stdout(session, chunk=cc, output=kernel_msg_content['text'])
+                        progress.session_chunk_stdout(session, chunk=cc, output=kernel_msg_content['text'])
                     elif kernel_msg_content['name'] == 'stderr':
                         cc.stderr_lines.extend(util.splitlines_lf(_home_path_re.sub('~', kernel_msg_content['text'])))
-                        progress.chunk_stderr(session, chunk=cc, output=kernel_msg_content['text'])
+                        progress.session_chunk_stderr(session, chunk=cc, output=kernel_msg_content['text'])
                 elif kernel_msg_type == 'error':
                     kernel_msg_text = _ansi_color_escape_code_re.sub('', '\n'.join(kernel_msg_content['traceback']))
                     kernel_msg_text = _home_path_re.sub('~', kernel_msg_text)
@@ -198,13 +198,11 @@ async def exec(session: Session, *, cache_key_path: pathlib.Path, progress: Prog
                     cc.stderr_lines.extend(util.splitlines_lf(kernel_msg_text))
                     cc.errors.append(message.StderrRunError(cc.stderr_lines))
                     kernel_has_errors = True
-                    progress.chunk_stderr(session, chunk=cc, output=kernel_msg_text)
+                    progress.session_chunk_stderr(session, chunk=cc, output=kernel_msg_text)
             if not incomplete_cc_stack:
-                progress.chunk_end(session, chunk=cc)
+                progress.session_chunk_end(session, chunk=cc)
             else:
-                # `progress` only takes first chunk but accounts for all
-                # chunks that are grouped together
-                progress.chunk_end(session, chunk=incomplete_cc_stack[0])
+                progress.session_chunk_end(session, chunk=incomplete_cc_stack[-1])
                 incomplete_cc_stack = []
     finally:
         kernel_client.stop_channels()

codebraid/converters/base.py

@@ -263,7 +263,8 @@ def _process_code_chunks(self):
             origin_paths_for_cache=self.origin_paths_for_cache,
             code_defaults=self.code_defaults,
             session_defaults=self.session_defaults,
-            progress=self._progress
+            progress=self._progress,
+            only_code_output=self.only_code_output,
         )
         self.code_processor.process()