diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3ce6ed6e9..542bee80db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,7 @@ jobs: outputs: container: ${{ steps.filter.outputs.container }} outputs: ${{ steps.filter.outputs.outputs }} + style: ${{ steps.filter.outputs.style }} steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # @v2 @@ -37,6 +38,8 @@ jobs: container: - '.github/**' - 'container/**' + style: + - 'tutorial_*.rst' outputs: - '.github/**' - 'container/**' @@ -47,12 +50,17 @@ jobs: needs: changes uses: ./.github/workflows/container.yml - outputs: - if: | - always() - && contains(needs.*.result, 'success') - && !contains(needs.*.result, 'failure') - && (needs.changes.outputs.outputs == 'true' - || startsWith(github.ref, 'refs/tags/')) - needs: [ changes, container ] - uses: ./.github/workflows/outputs.yml + # outputs: + # if: | + # !cancelled() + # && contains(needs.*.result, 'success') + # && !contains(needs.*.result, 'failure') + # && (needs.changes.outputs.outputs == 'true' + # || startsWith(github.ref, 'refs/tags/')) + # needs: [ changes, container ] + # uses: ./.github/workflows/outputs.yml + + style: + if: ${{ needs.changes.outputs.style == 'true' }} + needs: changes + uses: ./.github/workflows/style.yml diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml new file mode 100644 index 0000000000..31dc5db4d5 --- /dev/null +++ b/.github/workflows/style.yml @@ -0,0 +1,19 @@ +name: Linting & Style Checks +on: + workflow_call: + + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + + - name: Set up Python + uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 + with: + python-version: '3.13' + + - name: Run RST Formatter + run: | + bin/format_rst_file.py --check tutorial_*.rst diff --git a/Makefile b/Makefile index 3523aa475f..f2e453377b 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,9 @@ clean: -rm -f command_index.rst -rm -rf $(BUILDDIR)/* $(APIDOC_FILES) +format: + bin/format_rst_file.py tutorial_*.rst + html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @@ -119,13 +122,13 @@ htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." + ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Spack.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Spack.qhc" @@ -149,7 +152,7 @@ latex: @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." + "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @@ -172,7 +175,7 @@ texinfo: @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." + "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @@ -194,12 +197,12 @@ linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." + "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." + "results in $(BUILDDIR)/doctest/output.txt." dashdoc: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/dashdoc diff --git a/bin/format_rst_file.py b/bin/format_rst_file.py new file mode 100755 index 0000000000..86a91a9efd --- /dev/null +++ b/bin/format_rst_file.py @@ -0,0 +1,683 @@ +#!/usr/bin/env python3 +""" +RST File Sentence Formatter + +A tool to reformat RST files by joining sentences split across multiple lines +and ensuring each sentence is on its own line, while preserving RST structure. +""" + +import argparse +import re +import sys +from pathlib import Path +from typing import List, Tuple, Optional + + +class RSTProcessor: + """Processes RST files to format sentences properly.""" + + def process_file(self, file_path: Path) -> bool: + """ + Process a single RST file. + + Args: + file_path: Path to the RST file to process + + Returns: + True if file was changed, False if already formatted correctly + """ + try: + # Read the original file + content = self._read_file(file_path) + if content is None: + return False + + # Process the content + processed_content = self._process_rst_content(content) + + # Check if content changed + if content == processed_content: + return False # No changes needed + + # Write back to the same file + if self._write_file(file_path, processed_content): + return True # File was changed + else: + return False + + except Exception as e: + print(f"error: cannot format {file_path}: {e}", file=sys.stderr) + return False + + def _read_file(self, file_path: Path) -> Optional[str]: + """Read file content safely.""" + try: + with open(file_path, "r", encoding="utf-8") as f: + return f.read() + except Exception as e: + print(f"error: cannot read {file_path}: {e}", file=sys.stderr) + return None + + def _write_file(self, file_path: Path, content: str) -> bool: + """Write file content safely.""" + try: + with open(file_path, "w", encoding="utf-8") as f: + f.write(content) + return True + except Exception as e: + print(f"error: cannot write {file_path}: {e}", file=sys.stderr) + return False + + def _process_rst_content(self, content: str) -> str: + """ + Process RST content to format sentences properly. + + Args: + content: Raw RST content + + Returns: + Processed RST content + """ + lines = content.split("\n") + result_lines = [] + in_license_header = False + in_directive_block = False + directive_indent = 0 + + # Check if we start with a license header + if lines and self._is_license_header_start(lines[0]): + in_license_header = True + + i = 0 + while i < len(lines): + line = lines[i] + + # Handle license header + if in_license_header: + result_lines.append(line) + if self._is_license_header_end(line, lines, i): + in_license_header = False + i += 1 + continue + + # Handle RST lists (process entire list at once) + if self._is_list_item(line): + list_lines, next_i = self._collect_list(lines, i) + result_lines.extend(list_lines) + i = next_i + continue + + # Handle RST tables (process entire table at once) + if self._is_table_line(line): + table_lines, next_i = self._collect_table(lines, i) + result_lines.extend(table_lines) + i = next_i + continue + + # Handle RST directive blocks (including code blocks) + if self._is_rst_directive_start(line): + in_directive_block = True + directive_indent = self._get_indent_level(line) + result_lines.append(line) + i += 1 + continue + + if in_directive_block: + if self._is_directive_block_end(line, directive_indent): + in_directive_block = False + else: + result_lines.append(line) + i += 1 + continue + + # Process regular content (only when not in special blocks) + if not in_license_header and not in_directive_block: + paragraph_lines, next_i = self._collect_paragraph(lines, i) + processed_lines = self._process_paragraph(paragraph_lines) + result_lines.extend(processed_lines) + i = next_i + else: + result_lines.append(line) + i += 1 + + return "\n".join(result_lines) + + def _is_license_header_start(self, line: str) -> bool: + """Check if line starts a license header.""" + return line.strip().startswith("..") and "Copyright" in line + + def _is_license_header_end(self, line: str, lines: List[str], index: int) -> bool: + """Check if license header ends.""" + return ( + line.strip() == "" + and index + 1 < len(lines) + and not lines[index + 1].strip().startswith("..") + ) + + def _is_list_item(self, line: str) -> bool: + """Check if line is an RST list item.""" + stripped = line.strip() + if not stripped: + return False + + # RST list patterns + list_patterns = [ + r"^\s*\*\s+", # Bullet list: * item + r"^\s*\+\s+", # Bullet list: + item + r"^\s*-\s+", # Bullet list: - item + r"^\s*\d+\.\s+", # Numbered list: 1. item + r"^\s*#\.\s+", # Auto-numbered list: #. item + r"^\s*\([a-zA-Z0-9]+\)\s+", # Parenthesized list: (a) item + r"^\s*[a-zA-Z]\.\s+", # Letter list: a. item + r"^\s*[IVX]+\.\s+", # Roman numeral list: I. item + ] + + return any(re.match(pattern, line) for pattern in list_patterns) + + def _collect_list(self, lines: List[str], start_idx: int) -> Tuple[List[str], int]: + """ + Collect all lines that are part of an RST list. + + Args: + lines: All lines in the document + start_idx: Starting index + + Returns: + Tuple of (list_lines, next_index) + """ + list_lines = [] + i = start_idx + base_indent = self._get_indent_level(lines[start_idx]) + + while i < len(lines): + line = lines[i] + + # If it's a list item at the same or deeper indentation, include it + if self._is_list_item(line): + current_indent = self._get_indent_level(line) + if current_indent >= base_indent: + list_lines.append(line) + i += 1 + continue + else: + # List item at shallower indentation, end current list + break + + # If it's an empty line, check if the list continues + if not line.strip(): + # Look ahead to see if list continues + if i + 1 < len(lines): + next_line = lines[i + 1] + if ( + self._is_list_item(next_line) + and self._get_indent_level(next_line) >= base_indent + ): + list_lines.append(line) # Include the empty line + i += 1 + continue + elif ( + next_line.strip() + and self._get_indent_level(next_line) > base_indent + ): + # Continuation of list item content + list_lines.append(line) + i += 1 + continue + # Empty line and no more list content, end list + break + + # If it's indented content (continuation of list item), include it + current_indent = self._get_indent_level(line) + if line.strip() and current_indent > base_indent: + list_lines.append(line) + i += 1 + continue + + # If it's not a list item, not empty, and not indented continuation, end list + break + + return list_lines, i + + def _is_table_line(self, line: str) -> bool: + """Check if line is part of an RST table.""" + stripped = line.strip() + if not stripped: + return False + + # Grid table patterns + # Lines made of =, -, +, and spaces (table borders) + if re.match(r"^[=\-+\s]+$", stripped) and len(stripped) > 3: + return True + + # Simple table patterns (lines with multiple spaces that could be column separators) + # But be more conservative - look for patterns that are clearly tabular + if " " in stripped and not stripped.startswith(".."): + # Check if it looks like a table row (has multiple column-like segments) + segments = [s.strip() for s in stripped.split(" ") if s.strip()] + if len(segments) >= 2: + return True + + return False + + def _collect_table(self, lines: List[str], start_idx: int) -> Tuple[List[str], int]: + """ + Collect all lines that are part of an RST table. + + Args: + lines: All lines in the document + start_idx: Starting index + + Returns: + Tuple of (table_lines, next_index) + """ + table_lines = [] + i = start_idx + + # Collect all consecutive table-related lines + while i < len(lines): + line = lines[i] + + # If it's a table line, include it + if self._is_table_line(line): + table_lines.append(line) + i += 1 + continue + + # If it's an empty line, check if the next line is also a table line + if not line.strip(): + # Look ahead to see if table continues + if i + 1 < len(lines) and self._is_table_line(lines[i + 1]): + table_lines.append(line) # Include the empty line + i += 1 + continue + else: + # Empty line and no more table content, end table + break + + # If it's not a table line and not empty, end table + break + + return table_lines, i + + def _is_rst_directive_start(self, line: str) -> bool: + """Check if line starts an RST directive that has indented content.""" + # Match any RST directive pattern (allowing hyphens in directive names) + return bool(re.match(r"^\s*\.\.\s+[\w-]+::", line)) + + def _is_directive_block_end(self, line: str, directive_indent: int) -> bool: + """Check if directive block ends.""" + if not line.strip(): + return False + current_indent = self._get_indent_level(line) + return current_indent <= directive_indent + + def _get_indent_level(self, line: str) -> int: + """Get the indentation level of a line.""" + return len(line) - len(line.lstrip()) + + def _collect_paragraph( + self, lines: List[str], start_idx: int + ) -> Tuple[List[str], int]: + """ + Collect lines that form a paragraph. + + Args: + lines: All lines in the document + start_idx: Starting index + + Returns: + Tuple of (paragraph_lines, next_index) + """ + if start_idx >= len(lines): + return [], start_idx + + current_line = lines[start_idx] + + # Handle special lines + if self._is_special_line(current_line) or not current_line.strip(): + return [current_line], start_idx + 1 + + # Collect continuation lines + paragraph_lines = [current_line] + base_indent = self._get_indent_level(current_line) + + i = start_idx + 1 + while i < len(lines): + line = lines[i] + + # Stop conditions + if ( + not line.strip() + or self._is_special_line(line) + or self._is_table_line(line) # Stop at table lines + or self._is_list_item(line) # Stop at list items + or abs(self._get_indent_level(line) - base_indent) > 2 + ): + break + + paragraph_lines.append(line) + i += 1 + + return paragraph_lines, i + + def _is_special_line(self, line: str) -> bool: + """Check if a line is a special RST construct.""" + stripped = line.strip() + + patterns = [ + r"^\.\.", # RST directives + r'^[=\-~^"#*+<>]{3,}$', # RST headers + r"^:", # RST fields + r"^\s*\.\.\s+_", # RST targets + ] + + return any(re.match(pattern, line) for pattern in patterns) + + def _process_paragraph(self, paragraph_lines: List[str]) -> List[str]: + """ + Process a paragraph by joining and splitting sentences. + + Args: + paragraph_lines: Lines that form a paragraph + + Returns: + Processed lines with proper sentence formatting + """ + if not paragraph_lines or len(paragraph_lines) == 1: + return paragraph_lines + + if all(self._is_special_line(line) for line in paragraph_lines): + return paragraph_lines + + # Join all lines + joined_text = self._join_paragraph_lines(paragraph_lines) + + if not joined_text: + return paragraph_lines + + # Split into sentences + sentences = self._split_into_sentences(joined_text) + + # Return sentences with no leading whitespace + if len(sentences) > 1: + return [sentence for sentence in sentences if sentence.strip()] + else: + return [joined_text] + + def _join_paragraph_lines(self, lines: List[str]) -> str: + """Join paragraph lines into a single text block.""" + joined_text = "" + for line in lines: + text = line.strip() + if text: + if joined_text and not joined_text.endswith(" "): + joined_text += " " + joined_text += text + return joined_text + + def _split_into_sentences(self, text: str) -> List[str]: + """Split text into sentences at sentence boundaries.""" + # Pattern for sentence endings + sentence_pattern = r"([.!?]+)(\s+)(?=[A-Z]|\s*$)" + parts = re.split(sentence_pattern, text) + + if len(parts) <= 1: + return [text] + + sentences = [] + current_sentence = "" + + i = 0 + while i < len(parts): + if i + 2 < len(parts) and re.match(r"[.!?]+", parts[i + 1]): + current_sentence += parts[i] + parts[i + 1] + sentences.append(current_sentence.strip()) + current_sentence = "" + i += 3 + else: + current_sentence += parts[i] + i += 1 + + if current_sentence.strip(): + sentences.append(current_sentence.strip()) + + return [s for s in sentences if s.strip()] + + +def is_rst_file(file_path: Path) -> bool: + """ + Check if a file is a reStructuredText file. + + Args: + file_path: Path to check + + Returns: + True if file appears to be RST, False otherwise + """ + # Check file extension first + rst_extensions = {".rst", ".rest", ".restx", ".rtxt"} + if file_path.suffix.lower() in rst_extensions: + return True + + # For files without RST extensions, check content + try: + with open(file_path, "r", encoding="utf-8") as f: + # Read first few lines to check for RST patterns + lines = [] + for _ in range(20): # Check first 20 lines + try: + line = next(f) + lines.append(line) + except StopIteration: + break + + content = "".join(lines) + + # Look for common RST patterns + rst_patterns = [ + r"^\.\. ", # RST directives + r'^[=\-~^"#*+<>]{3,}$', # RST headers/underlines + r"^\.\. _", # RST targets + r"^\.\. \|", # RST substitutions + r"::\s*$", # RST literal blocks + r"^\.\. code-block::", # Code blocks + r"^\.\. literalinclude::", # Literal includes + r"^\.\. note::", # Admonitions + r"^\.\. warning::", # Admonitions + r"^\.\. image::", # Images + r"^\.\. figure::", # Figures + ] + + # Count RST-specific patterns + rst_indicators = 0 + for line in lines: + line = line.strip() + for pattern in rst_patterns: + if re.match(pattern, line, re.MULTILINE): + rst_indicators += 1 + break + + # If we found multiple RST indicators, consider it an RST file + return rst_indicators >= 2 + + except (UnicodeDecodeError, IOError): + return False + + +def filter_rst_files(file_paths: List[Path]) -> Tuple[List[Path], List[Path]]: + """ + Filter files to only include RST files. + + Args: + file_paths: List of file paths to filter + + Returns: + Tuple of (rst_files, skipped_files) + """ + rst_files = [] + skipped_files = [] + + for file_path in file_paths: + if not file_path.exists(): + print( + f"error: cannot read {file_path}: No such file or directory", + file=sys.stderr, + ) + continue + + if not file_path.is_file(): + print(f"error: cannot read {file_path}: Not a file", file=sys.stderr) + continue + + if is_rst_file(file_path): + rst_files.append(file_path) + else: + skipped_files.append(file_path) + + return rst_files, skipped_files + + +def main() -> int: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Reformat RST files by ensuring each sentence is on its own line", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s file1.rst file2.rst file3.rst + %(prog)s *.rst + %(prog)s docs/*.rst + %(prog)s docs/ # Will find all RST files in directory + """, + ) + + parser.add_argument( + "files", nargs="+", type=Path, help="RST files or directories to process" + ) + + parser.add_argument( + "--check", + action="store_true", + help="Don't write the files back, just return the status. " + "Return code 0 means nothing would change. " + "Return code 1 means some files would be reformatted.", + ) + + parser.add_argument( + "--verbose", action="store_true", help="Show files that are skipped" + ) + + parser.add_argument("--version", action="version", version="%(prog)s 1.0.0") + + args = parser.parse_args() + + # Expand directories to find RST files + all_files = [] + for path in args.files: + if path.is_dir(): + # Find all potential RST files in directory + for ext in [".rst", ".rest", ".restx", ".rtxt"]: + all_files.extend(path.glob(f"**/*{ext}")) + # Also check files without extensions that might be RST + for file_path in path.rglob("*"): + if file_path.is_file() and not file_path.suffix: + all_files.append(file_path) + else: + all_files.append(path) + + # Filter to only RST files + rst_files, skipped_files = filter_rst_files(all_files) + + # Show skipped files if verbose + if args.verbose and skipped_files: + for file_path in skipped_files: + print(f"skipped: {file_path} (not a reStructuredText file)") + + if not rst_files: + if skipped_files: + print("No reStructuredText files found to format") + return 0 + + # Process RST files + processor = RSTProcessor() + changed_files = [] + unchanged_files = [] + error_files = [] + + for file_path in rst_files: + try: + # Read the original file + content = processor._read_file(file_path) + if content is None: + error_files.append(file_path) + continue + + # Process the content + processed_content = processor._process_rst_content(content) + + # Check if content changed + if content == processed_content: + unchanged_files.append(file_path) + else: + changed_files.append(file_path) + if not args.check: + # Write back to the same file + if not processor._write_file(file_path, processed_content): + error_files.append(file_path) + changed_files.remove(file_path) + + except Exception as e: + print(f"error: cannot format {file_path}: {e}", file=sys.stderr) + error_files.append(file_path) + + # Report results in black style + total_files = len(rst_files) + + if args.check: + if changed_files: + print( + f"would reformat {len(changed_files)} file{'s' if len(changed_files) != 1 else ''}" + ) + for file_path in changed_files: + print(f"would reformat {file_path}") + return 1 + else: + if total_files == 1: + print(f"{total_files} file left unchanged") + else: + print(f"{total_files} files left unchanged") + return 0 + else: + # Normal mode output + if changed_files: + for file_path in changed_files: + print(f"reformatted {file_path}") + + if unchanged_files and not changed_files: + # Only show "left unchanged" if no files were changed + if len(unchanged_files) == 1: + print(f"{len(unchanged_files)} file left unchanged") + else: + print(f"{len(unchanged_files)} files left unchanged") + elif unchanged_files and changed_files: + # Show summary when both changed and unchanged files exist + changed_count = len(changed_files) + unchanged_count = len(unchanged_files) + + parts = [] + if changed_count: + parts.append( + f"{changed_count} file{'s' if changed_count != 1 else ''} reformatted" + ) + if unchanged_count: + parts.append( + f"{unchanged_count} file{'s' if unchanged_count != 1 else ''} left unchanged" + ) + + print(", ".join(parts)) + + return 1 if error_files else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tutorial_advanced_packaging.rst b/tutorial_advanced_packaging.rst index 72e2957dc9..17edb0302b 100644 --- a/tutorial_advanced_packaging.rst +++ b/tutorial_advanced_packaging.rst @@ -11,17 +11,11 @@ Advanced Topics in Packaging ============================ -Spack tries to automatically configure packages with information from -dependencies such that all you need to do is to list the dependencies -(i.e., with the ``depends_on`` directive) and the build system (for example -by deriving from :code:`CmakePackage`). - -However, there are many special cases. Often you need to retrieve details -about dependencies to set package-specific configuration options, or to -define package-specific environment variables used by the package's build -system. This tutorial covers how to retrieve build information from -dependencies, and how you can automatically provide important information to -dependents in your package. +Spack tries to automatically configure packages with information from dependencies such that all you need to do is to list the dependencies (i.e., with the ``depends_on`` directive) and the build system (for example by deriving from :code:`CmakePackage`). + +However, there are many special cases. +Often you need to retrieve details about dependencies to set package-specific configuration options, or to define package-specific environment variables used by the package's build system. +This tutorial covers how to retrieve build information from dependencies, and how you can automatically provide important information to dependents in your package. ---------------------- Setup for the Tutorial @@ -32,26 +26,23 @@ Setup for the Tutorial We do not recommend doing this section of the tutorial in a production Spack instance. -The tutorial uses custom package definitions with missing sections that -will be filled in during the tutorial. These package definitions are stored -in a separate package repository, which can be enabled with: +The tutorial uses custom package definitions with missing sections that will be filled in during the tutorial. +These package definitions are stored in a separate package repository, which can be enabled with: .. code-block:: console $ spack repo add --scope=site var/spack/repos/tutorial -This section of the tutorial may also require a newer version of -gcc. If you have not already installed gcc @7.2.0 and added it to your -configuration, you can do so with: +This section of the tutorial may also require a newer version of gcc. +If you have not already installed gcc @7.2.0 and added it to your configuration, you can do so with: .. code-block:: console $ spack install gcc@7.2.0 %gcc@5.4.0 $ spack compiler add --scope=site `spack location -i gcc@7.2.0 %gcc@5.4.0` -If you are using the tutorial docker image, all dependency packages -will have been installed. Otherwise, to install these packages you can use -the following commands: +If you are using the tutorial docker image, all dependency packages will have been installed. +Otherwise, to install these packages you can use the following commands: .. code-block:: console @@ -59,8 +50,7 @@ the following commands: $ spack install netlib-lapack $ spack install mpich -Now, you are ready to set your preferred ``EDITOR`` and continue with -the rest of the tutorial. +Now, you are ready to set your preferred ``EDITOR`` and continue with the rest of the tutorial. .. note:: @@ -76,49 +66,33 @@ the rest of the tutorial. Modifying a Package's Build Environment --------------------------------------- -Spack sets up several environment variables like ``PATH`` by default to aid in -building a package, but many packages make use of environment variables which -convey specific information about their dependencies (e.g., ``MPICC``). -This section covers how to update your Spack packages so that package-specific -environment variables are defined at build-time. +Spack sets up several environment variables like ``PATH`` by default to aid in building a package, but many packages make use of environment variables which convey specific information about their dependencies (e.g., ``MPICC``). +This section covers how to update your Spack packages so that package-specific environment variables are defined at build-time. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Set environment variables in dependent packages at build-time ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Dependencies can set environment variables that are required when their -dependents build. For example, when a package depends on a python extension -like py-numpy, Spack's ``python`` package will add it to ``PYTHONPATH`` -so it is available at build time; this is required because the default setup -that spack does is not sufficient for python to import modules. +Dependencies can set environment variables that are required when their dependents build. +For example, when a package depends on a python extension like py-numpy, Spack's ``python`` package will add it to ``PYTHONPATH`` so it is available at build time; this is required because the default setup that spack does is not sufficient for python to import modules. To provide environment setup for a dependent, a package can implement the :py:func:`setup_dependent_build_environment -` -and or :py:func:`setup_dependent_run_environment -` functions. -These functions take as a parameter a :py:class:`EnvironmentModifications -` object, which includes -convenience methods to update the environment. For example, an MPI -implementation can set ``MPICC`` for build-time use for packages that -depend on it: +` and or :py:func:`setup_dependent_run_environment ` functions. +These functions take as a parameter a :py:class:`EnvironmentModifications ` object, which includes convenience methods to update the environment. +For example, an MPI implementation can set ``MPICC`` for build-time use for packages that depend on it: .. code-block:: python def setup_dependent_build_environment(self, env, dependent_spec): env.set('MPICC', join_path(self.prefix.bin, 'mpicc')) -In this case packages that depend on ``mpi`` will have ``MPICC`` defined in -their environment when they build. This section is focused on modifying the -build-time environment represented by ``env``, but it's worth noting that -modifications to the run-time environment, made through the +In this case packages that depend on ``mpi`` will have ``MPICC`` defined in their environment when they build. +This section is focused on modifying the build-time environment represented by ``env``, but it's worth noting that modifications to the run-time environment, made through the :py:func:`setup_dependent_run_environment -` function's -``env`` parameter, are included in Spack's automatically-generated -module files. +` function's ``env`` parameter, are included in Spack's automatically-generated module files. -We can practice by editing the ``mpich`` package to set the ``MPICC`` -environment variable in the build-time environment of dependent packages. +We can practice by editing the ``mpich`` package to set the ``MPICC`` environment variable in the build-time environment of dependent packages. .. code-block:: console @@ -140,8 +114,7 @@ Once you're finished, the method should look like this: spack_env.set('MPICH_F90', spack_fc) spack_env.set('MPICH_FC', spack_fc) -At this point we can, for instance, install ``netlib-scalapack`` with -``mpich``: +At this point we can, for instance, install ``netlib-scalapack`` with ``mpich``: .. code-block:: console @@ -158,8 +131,7 @@ At this point we can, for instance, install ``netlib-scalapack`` with [+] /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/netlib-scalapack-2.0.2-km7tsbgoyyywonyejkjoojskhc5knz3z -and double check the environment logs to verify that every variable was -set to the correct value. +and double check the environment logs to verify that every variable was set to the correct value. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Set environment variables in your own package @@ -167,8 +139,7 @@ Set environment variables in your own package Packages can modify their own build-time environment by implementing the :py:func:`setup_build_environment -` -function: +To contrast with ``qt``'s :py:func:`setup_dependent_build_environment ` function: .. code-block:: python @@ -208,8 +176,7 @@ function: env.set('QTLIB', self.prefix.lib) env.prepend_path('QT_PLUGIN_PATH', self.prefix.plugins) -It is not necessary to implement a ``setup_dependent_run_environment`` -method for ``qt`` so one is not provided. +It is not necessary to implement a ``setup_dependent_run_environment`` method for ``qt`` so one is not provided. Let's see how it works by completing the ``elpa`` package: @@ -238,23 +205,16 @@ At this point it's possible to proceed with the installation of ``elpa ^mpich`` Retrieving Library Information ------------------------------ -Although Spack attempts to help packages locate their dependency libraries -automatically (e.g. by setting ``PKG_CONFIG_PATH`` and ``CMAKE_PREFIX_PATH``), -a package may have unique configuration options that are required to locate -libraries. When a package needs information about dependency libraries, the -general approach in Spack is to query the dependencies for the locations of -their libraries and set configuration options accordingly. By default most -Spack packages know how to automatically locate their libraries. This section -covers how to retrieve library information from dependencies and how to locate -libraries when the default logic doesn't work. +Although Spack attempts to help packages locate their dependency libraries automatically (e.g. by setting ``PKG_CONFIG_PATH`` and ``CMAKE_PREFIX_PATH``), a package may have unique configuration options that are required to locate libraries. +When a package needs information about dependency libraries, the general approach in Spack is to query the dependencies for the locations of their libraries and set configuration options accordingly. +By default most Spack packages know how to automatically locate their libraries. +This section covers how to retrieve library information from dependencies and how to locate libraries when the default logic doesn't work. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Accessing dependency libraries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you need to access the libraries of a dependency, you can do so -via the ``libs`` property of the spec, for example in the ``arpack-ng`` -package: +If you need to access the libraries of a dependency, you can do so via the ``libs`` property of the spec, for example in the ``arpack-ng`` package: .. code-block:: python @@ -267,20 +227,16 @@ package: '-DBLAS_LIBRARIES={0}'.format(blas_libs) ], '.') -Note that ``arpack-ng`` is querying virtual dependencies, which Spack -automatically resolves to the installed implementation (e.g. ``openblas`` -for ``blas``). +Note that ``arpack-ng`` is querying virtual dependencies, which Spack automatically resolves to the installed implementation (e.g. ``openblas`` for ``blas``). -We've started work on a package for ``armadillo``. You should open it, -read through the comment that starts with ``# TUTORIAL:`` and complete -the ``cmake_args`` section: +We've started work on a package for ``armadillo``. +You should open it, read through the comment that starts with ``# TUTORIAL:`` and complete the ``cmake_args`` section: .. code-block:: console root@advanced-packaging-tutorial:/# spack edit armadillo -If you followed the instructions in the package, when you are finished your -``cmake_args`` method should look like: +If you followed the instructions in the package, when you are finished your ``cmake_args`` method should look like: .. code-block:: python @@ -301,12 +257,10 @@ If you followed the instructions in the package, when you are finished your '-DDETECT_HDF5={0}'.format('ON' if '+hdf5' in spec else 'OFF') ] -As you can see, getting the list of libraries that your dependencies provide -is as easy as accessing the their ``libs`` attribute. Furthermore, the interface -remains the same whether you are querying regular or virtual dependencies. +As you can see, getting the list of libraries that your dependencies provide is as easy as accessing the their ``libs`` attribute. +Furthermore, the interface remains the same whether you are querying regular or virtual dependencies. -At this point you can complete the installation of ``armadillo`` using ``openblas`` -as a LAPACK provider (``armadillo ^openblas ^mpich``): +At this point you can complete the installation of ``armadillo`` using ``openblas`` as a LAPACK provider (``armadillo ^openblas ^mpich``): .. code-block:: console @@ -327,22 +281,16 @@ as a LAPACK provider (``armadillo ^openblas ^mpich``): Fetch: 0.01s. Build: 3.96s. Total: 3.98s. [+] /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/armadillo-8.100.1-n2eojtazxbku6g4l5izucwwgnpwz77r4 -Hopefully the installation went fine and the code we added expanded to the right list -of semicolon separated libraries (you are encouraged to open ``armadillo``'s -build logs to double check). +Hopefully the installation went fine and the code we added expanded to the right list of semicolon separated libraries (you are encouraged to open ``armadillo``'s build logs to double check). ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Providing libraries to dependents ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Spack provides a default implementation for ``libs`` which often works -out of the box. A user can write a package definition without having to -implement a ``libs`` property and dependents can retrieve its libraries -as shown in the above section. However, the default implementation assumes that -libraries follow the naming scheme ``lib.so`` (or e.g. -``lib.a`` for static libraries). Packages which don't -follow this naming scheme must implement this function themselves, e.g. -``opencv``: +Spack provides a default implementation for ``libs`` which often works out of the box. +A user can write a package definition without having to implement a ``libs`` property and dependents can retrieve its libraries as shown in the above section. +However, the default implementation assumes that libraries follow the naming scheme ``lib.so`` (or e.g. ``lib.a`` for static libraries). +Packages which don't follow this naming scheme must implement this function themselves, e.g. ``opencv``: .. code-block:: python @@ -353,10 +301,8 @@ follow this naming scheme must implement this function themselves, e.g. "libopencv_*", root=self.prefix, shared=shared, recursive=True ) -This issue is common for packages which implement an interface (i.e. -virtual package providers in Spack). If we try to build another version of -``armadillo`` tied to ``netlib-lapack`` (``armadillo ^netlib-lapack ^mpich``) -we'll notice that this time the installation won't complete: +This issue is common for packages which implement an interface (i.e. virtual package providers in Spack). +If we try to build another version of ``armadillo`` tied to ``netlib-lapack`` (``armadillo ^netlib-lapack ^mpich``) we'll notice that this time the installation won't complete: .. code-block:: console @@ -385,9 +331,8 @@ we'll notice that this time the installation won't complete: See build log for details: /usr/local/var/spack/stage/arpack-ng-3.5.0-bloz7cqirpdxj33pg7uj32zs5likz2un/arpack-ng-3.5.0/spack-build-out.txt -Unlike ``openblas`` which provides a library named ``libopenblas.so``, -``netlib-lapack`` provides ``liblapack.so``, so it needs to implement -customized library search logic. Let's edit it: +Unlike ``openblas`` which provides a library named ``libopenblas.so``, ``netlib-lapack`` provides ``liblapack.so``, so it needs to implement customized library search logic. +Let's edit it: .. code-block:: console @@ -407,12 +352,9 @@ What we need to implement is: i.e., a property that returns the correct list of libraries for the LAPACK interface. -We use the name ``lapack_libs`` rather than ``libs`` because -``netlib-lapack`` can also provide ``blas``, and when it does it is provided -as a separate library file. Using this name ensures that when -dependents ask for ``lapack`` libraries, ``netlib-lapack`` will retrieve only -the libraries associated with the ``lapack`` interface. Now we can finally -install ``armadillo ^netlib-lapack ^mpich``: +We use the name ``lapack_libs`` rather than ``libs`` because ``netlib-lapack`` can also provide ``blas``, and when it does it is provided as a separate library file. +Using this name ensures that when dependents ask for ``lapack`` libraries, ``netlib-lapack`` will retrieve only the libraries associated with the ``lapack`` interface. +Now we can finally install ``armadillo ^netlib-lapack ^mpich``: .. code-block:: console @@ -427,10 +369,7 @@ install ``armadillo ^netlib-lapack ^mpich``: Fetch: 0.01s. Build: 3.75s. Total: 3.76s. [+] /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/armadillo-8.100.1-sxmpu5an4dshnhickh6ykchyfda7jpyn -Since each implementation of a virtual package is responsible for locating the -libraries associated with the interfaces it provides, dependents do not need -to include special-case logic for different implementations and for example -need only ask for :code:`spec['blas'].libs`. +Since each implementation of a virtual package is responsible for locating the libraries associated with the interfaces it provides, dependents do not need to include special-case logic for different implementations and for example need only ask for :code:`spec['blas'].libs`. ---------------------- Other Packaging Topics @@ -440,11 +379,9 @@ Other Packaging Topics Attach attributes to other packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Build tools usually also provide a set of executables that can be used -when another package is being installed. Spack gives you the opportunity -to monkey-patch dependent modules and attach attributes to them. This -helps make the packager experience as similar as possible to what would -have been the manual installation of the same package. +Build tools usually also provide a set of executables that can be used when another package is being installed. +Spack gives you the opportunity to monkey-patch dependent modules and attach attributes to them. +This helps make the packager experience as similar as possible to what would have been the manual installation of the same package. An example here is the ``automake`` package, which overrides :py:func:`setup_dependent_package `: @@ -458,8 +395,7 @@ An example here is the ``automake`` package, which overrides for name in executables: setattr(module, name, self._make_executable(name)) -so that every other package that depends on it can use directly ``aclocal`` -and ``automake`` with the usual function call syntax of :py:class:`Executable `: +so that every other package that depends on it can use directly ``aclocal`` and ``automake`` with the usual function call syntax of :py:class:`Executable `: .. code-block:: python @@ -469,10 +405,8 @@ and ``automake`` with the usual function call syntax of :py:class:`Executable 1 + 1 2 -Now we'd like to share these executables with other users. First we will focus -on sharing the binaries with other *Spack* users, and later we will see how -users completely unfamiliar with Spack can easily use the applications too. +Now we'd like to share these executables with other users. +First we will focus on sharing the binaries with other *Spack* users, and later we will see how users completely unfamiliar with Spack can easily use the applications too. ------------------------------------------------ Setting up an OCI build cache on GitHub Packages ------------------------------------------------ -For this tutorial we will be using GitHub Packages as an OCI registry, since -most people have a GitHub account and it's easy to use. +For this tutorial we will be using GitHub Packages as an OCI registry, since most people have a GitHub account and it's easy to use. -First go to ``_ to generate a Personal access -token with ``write:packages`` permissions. Copy this token. +First, go to ``_ to generate a Personal Access Token (classic) with ``write:packages`` permissions. +Copy this token. -Next, we will add this token to the mirror config section of the Spack environment: +Next, we will add this token to the mirror configuration section for the Spack environment. +Replace `` with your GitHub username and `` with your GitHub username or an organization where you have permission to create packages. +The build cache name, `buildcache-${USER}-${HOSTNAME}`, is a suggestion; you can choose your own. .. code-block:: console - $ export MY_OCI_TOKEN= + $ export MY_OCI_TOKEN= $ spack -e . mirror add \ - --oci-username \ + --oci-username \ --oci-password-variable MY_OCI_TOKEN \ --unsigned \ my-mirror \ - oci://ghcr.io//buildcache-${USER}-${HOSTNAME} + oci://ghcr.io//buildcache-${USER}-${HOSTNAME} .. note :: - We talk about mirrors and build caches almost interchangeably, because every build - cache is a binary mirror. Source mirrors exist too, which we will not cover in this - tutorial. +We talk about mirrors and build caches almost interchangeably, because every build cache is a binary mirror. +Source mirrors exist too, which we will not cover in this tutorial. Your ``spack.yaml`` file should now contain the following: @@ -110,7 +103,7 @@ which outputs ... ==> Pushed julia@1.9.3/dfzhutf to ghcr.io//buildcache--:julia-1.9.3-dfzhutfh3s2ekaltdmujjn575eip5uhl.spack -The location of the pushed package +The location of the pushed package, when referred to as a OCI image, will be: .. code-block:: text @@ -120,11 +113,9 @@ looks very similar to a container image --- we will get to that in a bit. .. note :: - Binaries pushed to GitHub packages are ``private`` by default, which means you need a token - to download them. You can change the visibility to ``public`` by going to GitHub Packages - from your GitHub account, selecting the ``buildcache`` package, go to ``package settings``, - and change the visibility to ``public`` in the ``Danger Zone`` section. This page can also - be directly accessed by going to +Binaries pushed to GitHub packages are ``private`` by default, which means you need a token to download them. +You can change the visibility to ``public`` by going to GitHub Packages from your GitHub account, selecting the ``buildcache`` package, go to ``package settings``, and change the visibility to ``public`` in the ``Danger Zone`` section. +This page can also be directly accessed by going to .. code-block:: text @@ -137,10 +128,8 @@ Installing from the build cache We will now verify that the build cache works by reinstalling ``julia``. -Let's make sure that we *only* use the build cache that we just created, and not the -builtin one that is configured for the tutorial. The easiest way to do this is to -override the ``mirrors`` config section in the environment by using a double colon -in the ``spack.yaml`` file: +Let's make sure that we *only* use the build cache that we just created, and not the builtin one that is configured for the tutorial. +The easiest way to do this is to override the ``mirrors`` config section in the environment by using a double colon in the ``spack.yaml`` file: .. code-block:: yaml @@ -155,7 +144,7 @@ in the ``spack.yaml`` file: secret_variable: MY_OCI_TOKEN signed: false -An "overwrite install" should be enough to show that the build cache is used: +An "overwrite install" should be enough to show that the build cache is used (output will vary based on your specific configuration): .. code-block:: console @@ -165,38 +154,34 @@ An "overwrite install" should be enough to show that the build cache is used: ==> Extracting julia-1.9.3-dfzhutfh3s2ekaltdmujjn575eip5uhl from binary cache [+] /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/julia-1.9.3-dfzhutfh3s2ekaltdmujjn575eip5uhl -Two blobs are fetched for each spec: a metadata file and the actual binary package. If you've -used ``docker pull`` or other container runtimes before, these types of hashes may look -familiar. OCI registries are content addressed, which means that we see hashes like these -instead of human-readable file names. +Two blobs are fetched for each spec: a metadata file and the actual binary package. +If you've used ``docker pull`` or other container runtimes before, these types of hashes may look familiar. +OCI registries are content addressed, which means that we see hashes like these instead of human-readable file names. ------------------------------------ Reuse of binaries from a build cache ------------------------------------ -Spack's concretizer optimizes for **reuse**. This means that it will avoid source builds if it -can use specs for which binaries are readily available. +Spack's concretizer optimizes for **reuse**. +This means that it will avoid source builds if it can use specs for which binaries are readily available. -In the previous example we managed to install packages from our build cache, but we did not -concretize our environment again. Users on other machines with different distributions will have -to concretize, and therefore we should make sure that the build cache is indexed so that the -concretizer can take it into account. This can be done by running +In the previous example we managed to install packages from our build cache, but we did not concretize our environment again. +Users on other machines with different distributions will have to concretize, and therefore we should make sure that the build cache is indexed so that the concretizer can take it into account. +This can be done by running .. code-block:: console $ spack -e . buildcache update-index my-mirror -This operation can take a while for large build caches, since it fetches all metadata of -available packages. For convenience you can also run ``spack buildcache push --update-index ...`` -to avoid a separate step. +This operation can take a while for large build caches, since it fetches all metadata of available packages. +For convenience you can also run ``spack buildcache push --update-index ...`` to avoid a separate step. .. note:: - As of Spack 0.22, build caches can be used across different Linux distros. The concretizer - will reuse specs that have a host compatible ``libc`` dependency (e.g. ``glibc`` or ``musl``). - For packages compiled with ``gcc`` (and a few others), users do not have to install compilers - first, as the build cache contains the compiler runtime libraries as a separate package. + As of Spack 0.22, build caches can be used across different Linux distros. + The concretizer will reuse specs that have a host-compatible ``libc`` dependency (e.g. ``glibc`` or ``musl``). + For packages compiled with ``gcc`` (and a few other compilers), users do not have to install compilers first, as the build cache contains the compiler runtime libraries as a separate package dependency. After an index is created, it's possible to list the available packages in the build cache: @@ -209,25 +194,22 @@ After an index is created, it's possible to list the available packages in the b Creating runnable container images ---------------------------------- -The build cache we have created uses an OCI registry, which is the same technology that is used -to store container images. So far we have used this build cache as any other build cache: the -concretizer can use it to avoid source builds, and ``spack install`` will fetch binaries from it. +The build cache we have created uses an OCI registry, which is the same technology that is used to store container images. +So far we have used this build cache as any other build cache: the concretizer can use it to avoid source builds, and ``spack install`` will fetch binaries from it. However, we can also use this build cache to share binaries directly as runnable container images. -We can already attempt to run the image associated with the ``julia`` package that we have -pushed earlier: +We can already attempt to run the image associated with the ``julia`` package that we have pushed earlier: .. code-block:: console $ docker run ghcr.io//buildcache--:julia-1.9.3-dfzhutfh3s2ekaltdmujjn575eip5uhl.spack julia exec /home/spack/spack/opt/spack/linux-ubuntu22.04-x86_64_v3/gcc-11.4.0/julia-1.9.3-dfzhutfh3s2ekaltdmujjn575eip5uhl/bin/julia: no such file or directory -but immediately we see it fails. The reason is that one crucial part is missing, and that is a -``glibc``, which Spack always treats as an external package. +but immediately we see it fails. +The reason is that one crucial part is missing, and that is a ``glibc``, which Spack always treats as an external package. -To fix this, we force push to the registry again, but this time we specify a base image with a -recent version of ``glibc``, for example from ``ubuntu:24.04``: +To fix this, we force push to the registry again, but this time we specify a base image with a recent version of ``glibc``, for example from ``ubuntu:24.04``: .. code-block:: console @@ -245,30 +227,25 @@ Now let's pull this image again and run it: julia> 1 + 1 2 -This time it works! The minimal ``ubuntu:24.04`` image provides us not only with ``glibc``, but -also other utilities like a shell. +This time it works! +The minimal ``ubuntu:24.04`` image provides us not only with ``glibc``, but also other utilities like a shell. -Notice that you can use any base image of choice, like ``fedora`` or ``rockylinux``. The only -constraint is that it has a ``libc`` compatible with the external in the Spack built the binaries. +Notice that you can use any base image of choice, like ``fedora`` or ``rockylinux``. +The only constraint is that it has a ``libc`` compatible with the external in the Spack built the binaries. Spack does not validate this. -------------------------------------- Spack environments as container images -------------------------------------- -The previous container image is a good start, but it would be nice to add some more utilities to -the image. If you've paid attention to the output of some of the commands we have run so far, -you may have noticed that Spack generates exactly one image tag for each package it pushes to -the registry. Every Spack package corresponds to a single layer in each image, and the layers -are shared across the different image tags. +The previous container image is a good start, but it would be nice to add some more utilities to the image. +If you've paid attention to the output of some of the commands we have run so far, you may have noticed that Spack generates exactly one image tag for each package it pushes to the registry. +Every Spack package corresponds to a single layer in each image, and the layers are shared across the different image tags. -Because Spack installs every package into a unique prefix, it is incredibly easy to compose -multiple packages into a container image. In contrast to Docker images built from commands -in a ``Dockerfile`` where each command is run in order, Spack package layers are independent, -and can in principle be combined in any order. +Because Spack installs every package into a unique prefix, it is incredibly easy to compose multiple packages into a container image. +In contrast to Docker images built from commands in a ``Dockerfile`` where each command is run in order, Spack package layers are independent, and can in principle be combined in any order. -Let's add a simple text editor like ``vim`` to our previous environment next to ``julia``, so that -we could both edit and run Julia code. +Let's add a simple text editor like ``vim`` to our previous environment next to ``julia``, so that we could both edit and run Julia code. .. note:: @@ -279,8 +256,7 @@ we could both edit and run Julia code. $ spack -e . install --add vim -This time we push to the OCI registry, but also pass ``--tag julia-and-vim`` to instruct Spack -to create an image for the environment as a whole, with a human-readable tag: +This time, when we push to the OCI registry, we also pass ``--tag julia-and-vim`` to instruct Spack to create an additional image tag for the environment as a whole, with a more human-readable name: .. code-block:: console @@ -300,14 +276,10 @@ Now let's run a container from this image: Do I need ``docker`` or ``buildah``? ------------------------------------ -In older versions of Spack it was common practice to generate a ``Dockerfile`` from a -Spack environment using the ``spack containerize`` command, and then use ``docker build`` -or other runtimes to create a container image. +In older versions of Spack it was common practice to generate a ``Dockerfile`` from a Spack environment using the ``spack containerize`` command, and then use ``docker build`` or other runtimes to create a container image. -This would trigger a multi-stage build, where the first stage would install Spack itself, -compilers and the environment, and the second stage would copy the installed environment -into a smaller image. For those familiar with ``Dockerfile`` syntax, it would structurally look -like this: +This would trigger a multi-stage build, where the first stage would install Spack itself, compilers and the environment, and the second stage would copy the installed environment into a smaller image. +For those familiar with ``Dockerfile`` syntax, it would structurally look like this: .. code-block:: Dockerfile @@ -318,44 +290,30 @@ like this: FROM COPY --from=build /opt/spack/opt /opt/spack/opt -This approach is still valid, and the ``spack containerize`` command continues to exist, but it -has a few downsides: +This approach is still valid, and the ``spack containerize`` command continues to exist, but it has a few downsides: -* When ``RUN spack -e /root/env install`` fails, ``docker`` will not cache the layer, meaning - that all dependencies that did install successfully are lost. Troubleshooting the build - typically means starting from scratch in ``docker run`` or on the host system. -* In certain CI environments, it is not possible to use ``docker build``. For example, the - CI script itself may already run in a docker container, and running ``docker build`` *safely* - inside a container is tricky. +* When ``RUN spack -e /root/env install`` fails, ``docker`` will not cache the layer, meaning that all dependencies that did install successfully are lost. Troubleshooting the build typically means starting from scratch either within a ``docker run`` session or on the host system. +* In certain CI environments, it is not possible to use ``docker build`` directly. For example, the CI script itself may already run in a Docker container, and running ``docker build`` *safely* inside a container (Docker-in-Docker) is tricky. -The takeaway is that Spack decouples the steps that ``docker build`` combines: -build isolation, running the build, and creating an image. You can run -``spack install`` on your host machine or in a container, and run -``spack buildcache push`` separately to create an image. +The takeaway is that Spack decouples the steps that ``docker build`` combines: build isolation, running the build, and creating an image. +You can run ``spack install`` on your host machine or in a container, and run ``spack buildcache push`` separately to create an image. ---------- Relocation ---------- -Spack is different from many package managers in that it lets users choose where to install -packages. This makes Spack very flexible, as users can install packages in their home directory -and do not need root privileges. The downside is that sharing binaries is more complicated, -as binaries may contain hard-coded, absolute paths to machine specific locations, which have -to be adjusted when binaries are installed on a different machine. +Spack is different from many package managers in that it lets users choose where to install packages. +This makes Spack very flexible, as users can install packages in their home directory and do not need root privileges. +The downside is that sharing binaries is more complicated, as binaries may contain hard-coded, absolute paths to machine specific locations, which have to be adjusted when these binaries are installed on a different machine or in a different path. -Fortunately Spack handles this automatically upon install from a binary cache. But when you -build binaries that are intended to be shared, there is one thing you have to keep in mind: -Spack can relocate hard-coded paths in binaries *provided that the target prefix is shorter -than the prefix used during the build*. +Fortunately Spack handles this automatically upon install from a binary cache. +But when you build binaries that are intended to be shared, there is one thing you have to keep in mind: Spack can relocate hard-coded paths in binaries *provided that the target prefix is shorter than the prefix used during the build*. -The reason is that binaries typically embed these absolute paths in string tables, which is -a list of null terminated strings, to which the program stores offsets. That means we can -only modify strings in-place, and if the new path is longer than the old one, we would -overwrite the next string in the table. +The reason is that binaries typically embed these absolute paths in string tables, which is a list of null terminated strings, to which the program stores offsets. +That means we can only modify strings in-place, and if the new path is longer than the old one, we would overwrite the next string in the table. -To maximize the chances of successful relocation, you should build your binaries in a -relative long path. Fortunately Spack can automatically pad paths to make them longer, -using the following command: +To maximize the chances of successful relocation, you should build your binaries in a relatively long path. +Fortunately Spack can automatically pad paths to make them longer, using the following command: .. code-block:: console @@ -365,11 +323,10 @@ using the following command: Using build caches in CI ------------------------ -Build caches are a great way to speed up CI pipelines. Both GitHub Actions and Gitlab CI -support container registries, and this tutorial should give you a good starting point to -leverage them. +Build caches are a great way to speed up CI pipelines. +Both GitHub Actions and GitLab CI support container registries, and this tutorial should give you a good starting point to leverage them. -Spack also provides a basic GitHub Action to already provide you with a binary cache: +Spack also provides a basic GitHub Action that already provides you with a binary cache: .. code-block:: yaml @@ -381,8 +338,7 @@ Spack also provides a basic GitHub Action to already provide you with a binary c uses: spack/setup-spack@v2 - run: spack install python # uses a shared build cache -and the `setup-spack readme `_ shows you how to cache further -binaries that are not in the shared build cache. +and the `setup-spack readme `_ shows you how to cache further binaries that are not in the shared build cache. ------- Summary @@ -390,6 +346,6 @@ Summary In this tutorial we have created a build cache on top of an OCI registry, which can be used -* to ``spack install julia vim`` on machines without source builds -* to automatically create container images for individual packages while pushing to the cache -* to create container images for multiple packages at once +* to run ``spack install julia vim`` on machines and have Spack fetch pre-built binaries instead of building from source. +* to automatically create container images for individual packages when pushing to the cache. +* to create container images for entire Spack environments (multiple packages) at once. diff --git a/tutorial_buildsystems.rst b/tutorial_buildsystems.rst index ae87cd871f..dff85f9116 100644 --- a/tutorial_buildsystems.rst +++ b/tutorial_buildsystems.rst @@ -11,16 +11,12 @@ Spack Package Build Systems =========================== -You may begin to notice after writing a couple of package template files a -pattern emerge for some packages. For example, you may find yourself writing -an :code:`install()` method that invokes: :code:`configure`, :code:`cmake`, -:code:`make`, :code:`make install`. You may also find yourself writing -:code:`"prefix=" + prefix` as an argument to :code:`configure` or :code:`cmake`. -Rather than having you repeat these lines for all packages, Spack has -classes that can take care of these patterns. In addition, -these package files allow for finer grained control of these build systems. -In this section, we will describe each build system and give examples on -how these can be manipulated to install a package. +You may begin to notice after writing a couple of package template files that a pattern emerges for some packages. +For example, you may find yourself writing an ``install()`` method that invokes: ``configure``, ``cmake``, ``make``, ``make install``. +You may also find yourself writing ``"prefix=" + prefix`` as an argument to ``configure`` or ``cmake``. +Rather than having you repeat these lines for all packages, Spack has classes that can take care of these patterns. +In addition, these package files allow for finer-grained control of these build systems. +In this section, we will describe each build system and give examples on how these can be used to simplify packaging. ----------------------- Package Class Hierarchy @@ -44,35 +40,26 @@ Package Class Hierarchy PackageBase -> PythonPackage [dir=back] } -The above diagram gives a high level view of the class hierarchy and how each -package relates. Each subclass inherits from the :code:`PackageBaseClass` -super class. The bulk of the work is done in this super class which includes -fetching, extracting to a staging directory and installing. Each subclass -then adds additional build-system-specific functionality. In the following -sections, we will go over examples of how to utilize each subclass and to see -how powerful these abstractions are when packaging. +The above diagram gives a high-level view of the class hierarchy and how each package relates. +Each build system specific class inherits from the ``PackageBaseClass`` superclass. +The bulk of the common work is done in this superclass which includes fetching, extracting to a staging directory and installing. +Each subclass then adds additional build-system-specific functionality. +In the following sections, we will go over examples of how to utilize each subclass and see how powerful these abstractions are when packaging. ----------------- Package ----------------- -We've already seen examples of a :code:`Package` class in our walkthrough for writing -package files, so we won't be spending much time with them here. Briefly, -the Package class allows for arbitrary control over the build process, whereas -subclasses rely on certain patterns (e.g. :code:`configure` :code:`make` -:code:`make install`) to be useful. :code:`Package` classes are particularly useful -for packages that have a non-conventional way of being built since the packager -can utilize some of Spack's helper functions to customize the building and -installing of a package. +We've already seen examples of using the generic ``Package`` class in our walkthrough for writing package files, so we won't be spending much time with it here. +Briefly, the Package class allows for arbitrary control over the build process, whereas subclasses rely on certain patterns (e.g. ``configure`` ``make`` ``make install``) to be useful. +The generic ``Package`` class is particularly useful for packages that have a non-conventional build process, as it allows the packager to use Spack's helper functions to customize the building and installing of a package fully. ------------------- Autotools ------------------- -As we have seen earlier, packages using :code:`Autotools` use :code:`configure`, -:code:`make` and :code:`make install` commands to execute the build and -install process. In our :code:`Package` class, your typical build incantation will -consist of the following: +As we have seen earlier, packages using ``Autotools`` use ``configure``, ``make`` and ``make install`` commands to execute the build and install process. +In our ``Package`` class, your typical build incantation will consist of the following: .. code-block:: python @@ -83,30 +70,28 @@ consist of the following: You'll see that this looks similar to what we wrote in our packaging tutorial. -The :code:`Autotools` subclass aims to simplify writing package files and provides -convenience methods to manipulate each of the different phases for a :code:`Autotools` -build system. +The ``AutotoolsPackage`` subclass aims to simplify writing package files for Autotools-based software and provides convenient methods to manipulate each of the different phases for an ``Autotools`` build system. -:code:`Autotools` packages consist of four phases: +``AutotoolsPackage`` builds consist of four main phases, each cooresponding to a method that can be overridden: -1. :code:`autoreconf()` -2. :code:`configure()` -3. :code:`build()` -4. :code:`install()` +1. ``autoreconf()`` +2. ``configure()`` +3. ``build()`` +4. ``install()`` -Each of these phases have sensible defaults. Let's take a quick look at some -the internals of the :code:`Autotools` class: +Each of these phases have sensible defaults. +Let's take a quick look at some of the internals of the ``Autotools`` class: .. code-block:: console $ spack edit --build-system autotools -This will open the :code:`AutotoolsPackage` file in your text editor. +This will open the ``AutotoolsPackage`` file in your text editor. .. note:: - The examples showing code for these classes is abridged to avoid having + The examples showing code for these classes are abridged to avoid having long examples. We only show what is relevant to the packager. @@ -116,90 +101,76 @@ This will open the :code:`AutotoolsPackage` file in your text editor. :linenos: -Important to note are the highlighted lines. These properties allow the -packager to set what build targets and install targets they want for their -package. If, for example, we wanted to add as our build target :code:`foo` -then we can append to our :code:`build_targets` property: +Important to note are the highlighted lines. +These properties allow the packager to set what build targets and install targets they want for their package. +If, for example, we wanted to add as our build target ``foo`` then we can append to our ``build_targets`` property: .. code-block:: python build_targets = ["foo"] -Which is similar to invoking make in our Package +Which is similar to invoking ``make`` in our Package .. code-block:: python make("foo") -This is useful if we have packages that ignore environment variables and need -a command-line argument. +This is useful if we have packages that ignore environment variables and need a command-line argument. -Another thing to take note of is in the :code:`configure()` method. -Here we see that the :code:`prefix` argument is already included since it is a -common pattern amongst packages using :code:`Autotools`. We then only have to -override :code:`configure_args()`, which will then return it's output to -to :code:`configure()`. Then, :code:`configure()` will append the common -arguments +Another thing to take note of is in the ``configure()`` method in ``AutotoolsPackage``. +Here we see that the ``--prefix`` argument is already included since it is a common pattern amongst packages using ``Autotools``. +Therefore, we typically only need to override the ``configure_args()`` method to return a list of additional arguments. +The ``configure()`` method will then append these to the standard arguments. -Packagers also have the option to run :code:`autoreconf` in case a package -needs to update the build system and generate a new :code:`configure`. Though, -for the most part this will be unnecessary. +Packagers also have the option to run ``autoreconf`` in case a package needs to update the build system and generate a new ``configure``. +However, for the most part this will be unnecessary. -Let's look at the :code:`mpileaks` package.py file that we worked on earlier: +Let's look at the ``mpileaks`` package.py file that we worked on earlier: .. code-block:: console $ spack edit mpileaks -Notice that mpileaks is a :code:`Package` class but uses the :code:`Autotools` -build system. Although this package is acceptable let's make this into an -:code:`AutotoolsPackage` class and simplify it further. +Notice that mpileaks was originally written as a generic ``Package`` but uses the ``Autotools`` build system. +Although this package is acceptable, let's covert it to an ``AutotoolsPackage`` to simplify it further. .. literalinclude:: tutorial/examples/Autotools/0.package.py :language: python :emphasize-lines: 9 :linenos: -We first inherit from the :code:`AutotoolsPackage` class. +We first inherit from the ``AutotoolsPackage`` class. -Although we could keep the :code:`install()` method, most of it can be handled -by the :code:`AutotoolsPackage` base class. In fact, the only thing that needs -to be overridden is :code:`configure_args()`. +Although we could keep the ``install()`` method, most of it can be handled by the ``AutotoolsPackage`` base class. +In fact, the only thing that needs to be overridden is ``configure_args()``. .. literalinclude:: tutorial/examples/Autotools/1.package.py :language: python :emphasize-lines: 25,26,27,28,29,30,31,32 :linenos: -Since Spack takes care of setting the prefix for us we can exclude that as -an argument to :code:`configure`. Our packages look simpler, and the packager -does not need to worry about whether they have properly included :code:`configure` -and :code:`make`. +Since Spack's ``AutotoolsPackage`` takes care of setting the prefix for us, we can exclude that as an argument to ``configure``. +Our package file looks simpler, and we don't not need to worry about whether we have properly included ``configure`` and ``make``. -This version of the :code:`mpileaks` package installs the same as the previous, -but the :code:`AutotoolsPackage` class lets us do it with a cleaner looking -package file. +This version of the ``mpileaks`` package installs the same as the previous, but the ``AutotoolsPackage`` class lets us do it with a cleaner looking package file. ----------------- Makefile ----------------- -Packages that utilize :code:`Make` or a :code:`Makefile` usually require you -to edit a :code:`Makefile` to set up platform and compiler specific variables. -These packages are handled by the :code:`Makefile` subclass which provides -convenience methods to help write these types of packages. +Packages that utilize ``Make`` or a ``Makefile`` usually require you to edit a ``Makefile`` to set up platform and compiler specific variables. +These packages are handled by the ``Makefile`` subclass which provides convenience methods to help write these types of packages. -A :code:`MakefilePackage` class has three phases that can be overridden. These include: +A ``MakefilePackage`` build has three phases that can be overridden by the packager: - 1. :code:`edit()` - 2. :code:`build()` - 3. :code:`install()` + 1. ``edit()`` + 2. ``build()`` + 3. ``install()`` -Packagers then have the ability to control how a :code:`Makefile` is edited, and -what targets to include for the build phase or install phase. +Packagers then have the ability to control how a ``Makefile`` is edited, and what targets to include for the build phase or install phase. -Let's also take a look inside the :code:`MakefilePackage` class: +Let's also take a look inside the ``MakefilePackage`` class: .. code-block:: console @@ -214,9 +185,8 @@ Take note of the following: :lines: 40-111 :linenos: -Similar to :code:`Autotools`, :code:`MakefilePackage` class has properties -that can be set by the packager. We can also override the different -methods highlighted. +Similar to ``Autotools``, ``MakefilePackage`` class has properties that can be set by the packager. +We can also override the different methods highlighted. Let's try to recreate the Bowtie_ package: @@ -241,14 +211,13 @@ Let's try to recreate the Bowtie_ package: ==> Created template for bowtie package ==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/bowtie/package.py -Once the fetching is completed, Spack will open up your text editor in the -usual fashion and create a template of a :code:`MakefilePackage` package.py. +Once the fetching is completed, Spack will open up your text editor in the usual fashion and create a template of a ``MakefilePackage`` package.py. .. literalinclude:: tutorial/examples/Makefile/0.package.py :language: python :linenos: -Spack was successfully able to detect that :code:`Bowtie` uses :code:`Make`. +Spack was successfully able to detect that ``Bowtie`` uses ``Make``. Let's add in the rest of our details for our package: .. literalinclude:: tutorial/examples/Makefile/1.package.py @@ -256,24 +225,20 @@ Let's add in the rest of our details for our package: :emphasize-lines: 10,11,13,14,18,20 :linenos: -As we mentioned earlier, most packages using a :code:`Makefile` have hard-coded -variables that must be edited. These variables are fine if you happen to not -care about setup or types of compilers used but Spack is designed to work with -any compiler. The :code:`MakefilePackage` subclass makes it easy to edit -these :code:`Makefiles` by having an :code:`edit()` method that -can be overridden. +As we mentioned earlier, most packages using a ``Makefile`` have hard-coded variables that must be edited. +These variables are fine if you happen to not care about setup or types of compilers used but Spack is designed to work with any compiler. +The ``MakefilePackage`` subclass makes it easy to edit these ``Makefiles`` by having an ``edit()`` method that can be overridden. -Let's take a look at the default :code:`Makefile` that :code:`Bowtie` provides. -If we look inside, we see that :code:`CC` and :code:`CXX` point to our GNU -compiler: +Let's take a look at the default ``Makefile`` that ``Bowtie`` provides. +If we look inside, we see that ``CC`` and ``CXX`` point to our GNU compiler: .. code-block:: console $ spack stage bowtie .. note:: - As usual make sure you have shell support activated with spack: - :code:`source /path/to/spack_root/spack/share/spack/setup-env.sh` + As usual make sure you have shell support activated with Spack: + ``source /path/to/spack_root/spack/share/spack/setup-env.sh`` .. code-block:: console @@ -290,19 +255,16 @@ compiler: LIBS = $(LDFLAGS) -lz HEADERS = $(wildcard *.h) -To fix this, we need to use the :code:`edit()` method to write our custom -:code:`Makefile`. +To fix this, we need to use the ``edit()`` method to modify the ``Makefile``. .. literalinclude:: tutorial/examples/Makefile/2.package.py :language: python :emphasize-lines: 23,24,25 :linenos: -Here we use a :code:`FileFilter` object to edit our :code:`Makefile`. It takes -in a regular expression and then replaces :code:`CC` and :code:`CXX` to whatever -Spack sets :code:`CC` and :code:`CXX` environment variables to. This allows us to -build :code:`Bowtie` with whatever compiler we specify through Spack's -:code:`spec` syntax. +Here we use a ``FileFilter`` object (a Spack utility) to edit our ``Makefile``. +It takes a regular expression to find lines (e.g., assignments to ``CC`` and ``CXX``) and then replaces them with values derived from Spack's build environment (e.g., ``self.compiler.cc`` and ``self.compiler.cxx``). +This allows us to build ``Bowtie`` with whatever compiler we specify through Spack's spec syntax. Let's change the build and install phases of our package: @@ -311,14 +273,12 @@ Let's change the build and install phases of our package: :emphasize-lines: 28,29,30,31,32,35,36 :linenos: -Here demonstrate another strategy that we can use to manipulate our package -We can provide command-line arguments to :code:`make()`. Since :code:`Bowtie` -can use :code:`tbb` we can either add :code:`NO_TBB=1` as a argument to prevent -:code:`tbb` support or we can just invoke :code:`make` with no arguments. +Here we demonstrate another strategy that we can use to manipulate our package's build. +We can provide command-line arguments to ``make()``. +Since ``Bowtie`` can use ``tbb`` we can either add ``NO_TBB=1`` as a argument to prevent ``tbb`` support, or we can invoke ``make`` with no arguments if EBB is desired and found by its build system. -:code:`Bowtie` requires our :code:`install_target` to provide a path to -the install directory. We can do this by providing :code:`prefix=` as a command -line argument to :code:`make()`. +``Bowtie`` requires our ``install_target`` to provide a path to the install directory. +We can do this by providing ``prefix=`` as a command line argument to ``make()``. Let's look at a couple of other examples and go through them: @@ -327,9 +287,8 @@ Let's look at a couple of other examples and go through them: $ spack edit esmf Some packages allow environment variables to be set and will honor them. -Packages that use :code:`?=` for assignment in their :code:`Makefile` -can be set using environment variables. In our :code:`esmf` example we -set two environment variables in our :code:`edit()` method: +Packages that use ``?=`` for assignment in their ``Makefile`` can be set using environment variables. +In our ``esmf`` example we set two environment variables in our ``edit()`` method: .. code-block:: python @@ -355,17 +314,13 @@ set two environment variables in our :code:`edit()` method: msg += "'{0}', is not supported by ESMF." raise InstallError(msg.format(self.compiler.name)) -As you may have noticed, we didn't really write anything to the :code:`Makefile` -but rather we set environment variables that will override variables set in -the :code:`Makefile`. +As you may have noticed, we didn't really write anything to the ``Makefile`` but rather we set environment variables that will override variables set in the ``Makefile``. -Some packages include a configuration file that sets certain compiler variables, -platform specific variables, and the location of dependencies or libraries. -If the file is simple and only requires a couple of changes, we can overwrite -those entries with our :code:`FileFilter` object. If the configuration involves -complex changes, we can write a new configuration file from scratch. +Some packages include a configuration file that sets certain compiler variables, platform specific variables, and the location of dependencies or libraries. +If the file is simple and only requires a couple of changes, we can replace those entries with our ``FileFilter`` object. +If the configuration involves complex changes, we can write a new configuration file from scratch within the ``edit()`` method. -Let's look at an example of this in the :code:`elk` package: +Let's look at an example of this in the ``elk`` package: .. code-block:: console @@ -453,17 +408,15 @@ Let's look at an example of this in the :code:`elk` package: for key in config: inc.write('{0} = {1}\n'.format(key, config[key])) -:code:`config` is just a dictionary that we can add key-value pairs to. By the -end of the :code:`edit()` method we write the contents of our dictionary to -:code:`make.inc`. +``config`` is just a Python dictionary that we populate with key-value pairs. +By the end of the ``edit()`` method, we write the contents of our dictionary to the ``make.inc`` file, which the package's ``Makefile`` then includes. --------------- CMake --------------- -CMake_ is another common build system that has been gaining popularity. It works -in a similar manner to :code:`Autotools` but with differences in variable names, -the number of configuration options available, and the handling of shared libraries. +CMake_ is another common build system that has been gaining popularity. +It works in a similar manner to ``Autotools`` but with differences in variable names, the number of configuration options available, and the handling of shared libraries. Typical build incantations look like this: .. _CMake: https://cmake.org @@ -475,21 +428,19 @@ Typical build incantations look like this: make() make("install") -As you can see from the example above, it's very similar to invoking -:code:`configure` and :code:`make` in an :code:`Autotools` build system. However, -the variable names and options differ. Most options in CMake are prefixed -with a :code:`'-D'` flag to indicate a configuration setting. +As you can see from the example above, it's very similar to invoking ``configure`` and ``make`` in an ``Autotools`` build system. +However, the variable names and options differ. +Most options in CMake are prefixed with a ``'-D'`` flag to indicate a configuration setting. -In the :code:`CMakePackage` class we can override the following phases: +In the ``CMakePackage`` class, we can override the following build phases: -1. :code:`cmake()` -2. :code:`build()` -3. :code:`install()` +1. ``cmake()`` +2. ``build()`` +3. ``install()`` -The :code:`CMakePackage` class also provides sensible defaults so we only need to -override :code:`cmake_args()`. +The ``CMakePackage`` class also provides sensible defaults, so often we only need to override ``cmake_args()`` to pass package-specific options. -Let's look at these defaults in the :code:`CMakePackage` class in the :code:`_std_args()` method: +Let's look at these defaults in the ``CMakePackage`` class in the ``_std_args()`` method: .. code-block:: console @@ -501,46 +452,42 @@ Let's look at these defaults in the :code:`CMakePackage` class in the :code:`_st :emphasize-lines: 87,96 :linenos: -Some :code:`CMake` packages use different generators. Spack is able to support -Unix-Makefile_ generators as well as Ninja_ generators. +Some ``CMake`` packages use different generators. +Spack is able to support Unix-Makefile_ generators as well as Ninja_ generators. .. _Unix-Makefile: https://cmake.org/cmake/help/v3.4/generator/Unix%20Makefiles.html .. _Ninja: https://cmake.org/cmake/help/v3.4/generator/Ninja.html -If no generator is specified Spack will default to :code:`Unix Makefiles`. +If no generator is specified, Spack will default to ``Unix Makefiles``. -Next we setup the build type. In :code:`CMake` you can specify the build type -that you want. Options include: +Next we setup the build type. +In ``CMake`` you can specify the build type that you want. +Options include: -1. :code:`empty` -2. :code:`Debug` -3. :code:`Release` -4. :code:`RelWithDebInfo` -5. :code:`MinSizeRel` +1. ``empty`` +2. ``Debug`` +3. ``Release`` +4. ``RelWithDebInfo`` +5. ``MinSizeRel`` -With these options you can specify whether you want your executable to have -the debug version only, release version or the release with debug information. -Release executables tend to be more optimized than Debug. In Spack, we set -the default as Release unless otherwise specified through a variant. +With these options you can specify whether you want your executable to have the debug version only, release version or the release with debug information. +Release executables tend to be more optimized than Debug versions. +In Spack, we set the default as Release unless otherwise specified through a variant (e.g., ``build_type=Debug``). -Spack then automatically sets up the :code:`-DCMAKE_INSTALL_PREFIX` path, -appends the build type (:code:`RelWithDebInfo` default), and then specifies a verbose -:code:`Makefile`. +Spack then automatically sets up the ``-DCMAKE_INSTALL_PREFIX`` path, appends the build type (defaulting to ``RelWithDebInfo``), and enables a verbose ``Makefile`` output by default. -Next we add the :code:`rpaths` to :code:`-DCMAKE_INSTALL_RPATH:STRING`. +Next we add the ``rpaths`` to ``-DCMAKE_INSTALL_RPATH:STRING``. -Finally we add to :code:`-DCMAKE_PREFIX_PATH:STRING` the locations of all our -dependencies so that :code:`CMake` can find them. +Finally we add to ``-DCMAKE_PREFIX_PATH:STRING`` the locations of all our dependencies so that ``CMake`` can find them. -In the end our :code:`cmake` line will look like this (example is :code:`xrootd`): +In the end our ``cmake`` line will look like this (example is ``xrootd``): .. code-block:: console $ cmake $HOME/spack/var/spack/stage/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/xrootd-4.6.0 -G Unix Makefiles -DCMAKE_INSTALL_PREFIX:PATH=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_FIND_FRAMEWORK:STRING=LAST -DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE -DCMAKE_INSTALL_RPATH:STRING=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/lib:$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/lib64 -DCMAKE_PREFIX_PATH:STRING=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/cmake-3.9.4-hally3vnbzydiwl3skxcxcbzsscaasx5 -We can see now how :code:`CMake` takes care of a lot of the boilerplate code -that would have to be otherwise typed in. +We can see now how ``CMake`` takes care of a lot of the boilerplate code that would have to be otherwise typed in. Let's try to recreate callpath_: @@ -580,27 +527,23 @@ Again we fill in the details: :linenos: :emphasize-lines: 9,13,14,18,19,20,21,22,23 -As mentioned earlier, Spack will use sensible defaults to prevent repeated code -and to make writing :code:`CMake` package files simpler. +As mentioned earlier, Spack's ``CMakePackage`` uses sensible defaults to reduce biolerplate and simplify writing package files for ``CMake``-based software. -In callpath, we want to add options to :code:`CALLPATH_WALKER` as well as add -compiler flags. We add the following options like so: +In ``callpath``, we want to control options like ``CALLPATH_WALKER`` or add specific compiler flags. +We can return these options from ``cmake_args()`` like so: .. literalinclude:: tutorial/examples/Cmake/2.package.py :language: python :linenos: :emphasize-lines: 26,30,31 -Now we can control our build options using :code:`cmake_args()`. If defaults are -sufficient enough for the package, we can leave this method out. +Now we can control our build options using ``cmake_args()``. +If defaults are sufficient enough for the package, we can leave this method out. -:code:`CMakePackage` classes allow for control of other features in the -build system. For example, you can specify the path to the "out of source" -build directory and also point to the root of the :code:`CMakeLists.txt` file if it -is placed in a non-standard location. +``CMakePackage`` classes allow for control of other features in the build system. +For example, you can specify the path to the "out of source" build directory and also point to the root of the ``CMakeLists.txt`` file if it is placed in a non-standard location. -A good example of a package that has its :code:`CMakeLists.txt` file located at a -different location is found in :code:`spades`. +A good example of a package that has its ``CMakeLists.txt`` file located at a different location is found in ``spades``. .. code-block:: console @@ -610,19 +553,17 @@ different location is found in :code:`spades`. root_cmakelists_dir = "src" -Here :code:`root_cmakelists_dir` will tell Spack where to find the location -of :code:`CMakeLists.txt`. In this example, it is located a directory level below in -the :code:`src` directory. +Here ``root_cmakelists_dir`` will tell Spack where to find the location of ``CMakeLists.txt``. +In this example, it is located a directory level below in the ``src`` directory. -Some :code:`CMake` packages also require the :code:`install` phase to be -overridden. For example, let's take a look at :code:`sniffles`. +Some ``CMake`` packages also require the ``install`` phase to be overridden. +For example, let's take a look at ``sniffles``. .. code-block:: console $ spack edit sniffles -In the :code:`install()` method, we have to manually install our targets -so we override the :code:`install()` method to do it for us: +In the ``install()`` method, we have to manually install our targets so we override the ``install()`` method to do it for us: .. code-block:: python @@ -647,10 +588,8 @@ These modules are usually installed using the following line: $ pip install . -We can write package files for Python packages using the :code:`Package` class, -but the class brings with it a lot of methods that are useless for Python packages. -Instead, Spack has a :code:`PythonPackage` subclass that allows packagers -of Python modules to be able to invoke :code:`pip`. +We can write package files for Python packages using the ``Package`` class, but the class brings with it a lot of methods that are useless for Python packages. +Instead, Spack has a ``PythonPackage`` subclass that allows packagers of Python modules to be able to invoke ``pip``. We will write a package file for Pandas_: @@ -681,15 +620,15 @@ And we are left with the following template: :language: python :linenos: -As you can see this is not any different than any package template that we have -written. We have the choice of providing build options or using the sensible -defaults +As you can see this is not any different than any package template that we have written. +We have the choice of providing build options or using the sensible defaults Luckily for us, there is no need to provide build args. -Next we need to find the dependencies of a package. Dependencies are usually -listed in :code:`setup.py`. You can find the dependencies by searching for -:code:`install_requires` keyword in that file. Here it is for :code:`Pandas`: +Next we need to find the dependencies of a package. +Dependencies are usually listed in ``setup.py``. +You can find the dependencies by searching for ``install_requires`` keyword in that file. +Here it is for ``Pandas``: .. code-block:: python @@ -714,45 +653,37 @@ You can find a more comprehensive list at the Pandas documentation_. .. _documentation: https://pandas.pydata.org/pandas-docs/stable/install.html -By reading the documentation and :code:`setup.py` we found that :code:`Pandas` -depends on :code:`python-dateutil`, :code:`pytz`, and :code:`numpy`, :code:`numexpr`, -and finally :code:`bottleneck`. +By reading the documentation and ``setup.py`` we found that ``Pandas`` depends on ``python-dateutil``, ``pytz``, and ``numpy``, ``numexpr``, and finally ``bottleneck``. -Here is the completed :code:`Pandas` script: +Here is the completed ``Pandas`` script: .. literalinclude:: tutorial/examples/PyPackage/1.package.py :language: python :linenos: It is quite important to declare all the dependencies of a Python package. -Spack can "activate" Python packages to prevent the user from having to -load each dependency module explicitly. If a dependency is missed, Spack will -be unable to properly activate the package and it will cause an issue. To -learn more about extensions go to `spack extensions `_. +Spack can "activate" Python packages to prevent the user from having to load each dependency module explicitly. +If a dependency is missed, Spack will be unable to properly activate the package and it will cause an issue. +To learn more about extensions go to `spack extensions `_. -From this example, you can see that building Python modules is made easy -through the :code:`PythonPackage` class. +From this example, you can see that building Python modules is made easy through the ``PythonPackage`` class. ------------------- Other Build Systems ------------------- -Although we won't get in depth with any of the other build systems that Spack -supports, it is worth mentioning that Spack does provide subclasses -for the following build systems: +Although we won't get in depth with any of the other build systems that Spack supports, it is worth mentioning that Spack does provide subclasses for the following build systems: -1. :code:`IntelPackage` -2. :code:`SconsPackage` -3. :code:`WafPackage` -4. :code:`RPackage` -5. :code:`PerlPackage` -6. :code:`QMakePackage` +1. ``IntelPackage`` +2. ``SconsPackage`` +3. ``WafPackage`` +4. ``RPackage`` +5. ``PerlPackage`` +6. ``QMakePackage`` -Each of these classes have their own abstractions to help assist in writing -package files. For whatever doesn't fit nicely into the other build-systems, -you can use the :code:`Package` class. +Each of these classes have their own abstractions to help assist in writing package files. +For whatever doesn't fit nicely into the other build-systems, you can use the ``Package`` class. -Hopefully by now you can see how we aim to make packaging simple and -robust through these classes. If you want to learn more about these build -systems, check out `Implementing the installation procedure `_ in the Packaging Guide. +Hopefully by now you can see how we aim to make packaging simple and robust through these classes. +If you want to learn more about these build systems, check out `Implementing the installation procedure `_ in the Packaging Guide. diff --git a/tutorial_configuration.rst b/tutorial_configuration.rst index 26bea9608f..1c6f786259 100644 --- a/tutorial_configuration.rst +++ b/tutorial_configuration.rst @@ -11,9 +11,7 @@ Configuration Tutorial ====================== -This tutorial will guide you through various configuration options -that allow you to customize Spack's behavior with respect to -software installation. +This tutorial will guide you through various configuration options that allow you to customize Spack's behavior with respect to software installation. There are many different configuration sections. A partial list of some key configuration sections is provided below. @@ -37,8 +35,7 @@ A partial list of some key configuration sections is provided below. - Naming, location and additional configuration of Spack generated modules The full list of sections can be viewed with ``spack config list``. -For further education we encourage you to explore the spack -`documentation on configuration files `_. +For further education we encourage you to explore the spack `documentation on configuration files `_. The principle goals of this section of the tutorial are: @@ -46,62 +43,47 @@ The principle goals of this section of the tutorial are: 2. Demonstrate how to manipulate configurations 3. Show how to configure system assets with spack (compilers and packages) -As such we will primarily focus on the ``compilers`` -and ``packages`` configuration sections in this portion of the tutorial. +As such we will primarily focus on the ``compilers`` and ``packages`` configuration sections in this portion of the tutorial. -We will explain this by first covering how to manipulate configurations from -the command line and then show how this impacts the configuration file -hierarchy. We will then move into compiler and package configurations to help -you develop skills for getting the builds you want on your system. Finally, -we will give some brief attention to more generalized spack configurations -in the ``config`` section. +We will explain this by first covering how to manipulate configurations from the command line and then show how this impacts the configuration file hierarchy. +We will then move into compiler and package configurations to help you develop skills for getting the builds you want on your system. +Finally, we will give some brief attention to more generalized spack configurations in the ``config`` section. -For all of these features, we will demonstrate how we build up a full -configuration file. For some, we will then demonstrate how the -configuration affects the install command, and for others we will use -the ``spack spec`` command to demonstrate how the configuration -changes have affected Spack's concretization algorithm. The provided -output is all from a server running Ubuntu version 22.04. +For all of these features, we will demonstrate how we build up a full configuration file. +For some, we will then demonstrate how the configuration affects the install command, and for others we will use the ``spack spec`` command to demonstrate how the configuration changes have affected Spack's concretization algorithm. +The provided output is all from a server running Ubuntu version 22.04. ----------------------------------- Configuration from the command line ----------------------------------- -You can run ``spack config blame [section]`` at any point in time to see what -your current configuration is. If you omit the section then spack will dump all -the configurations settings to your screen. Let's go ahead and run this for the -``concretizer`` section. +You can run ``spack config blame [section]`` at any point in time to see what your current configuration is. +If you omit the section then spack will dump all the configurations settings to your screen. +Let's go ahead and run this for the ``concretizer`` section. .. code-block:: console $ spack config blame concretizer Notice that the ``spack:concretizer:reuse`` option is defaulted to ``true``. -For this section we'd actually like to turn reuse off so that when we demonstrate -package configuration our preferences are weighted higher than available binaries -for the concretizer solution selection procedure. +For this section we'd actually like to turn reuse off so that when we demonstrate package configuration our preferences are weighted higher than available binaries for the concretizer solution selection procedure. -One of the most convenient ways to set configuration options is -through the command line. +One of the most convenient ways to set configuration options is through the command line. .. code-block:: console $ spack config add concretizer:reuse:false -If we rerun ``spack config blame concretizer`` we can see that the change was -applied. +If we rerun ``spack config blame concretizer`` we can see that the change was applied. .. code-block:: console $ spack config blame concretizer Notice that the reference file on for this option is now different. -This indicates the scope where the configuration was set in, and we will -discuss how spack chooses the default scope shortly. -For now, it is important to note that the ``spack config`` command accepts an -optional ``--scope`` flag so we can be more precise in the configuration process. -This will make more sense after the next section which provides -the definition of spack's configuration scopes and their hierarchy. +This indicates the scope where the configuration was set in, and we will discuss how spack chooses the default scope shortly. +For now, it is important to note that the ``spack config`` command accepts an optional ``--scope`` flag so we can be more precise in the configuration process. +This will make more sense after the next section which provides the definition of spack's configuration scopes and their hierarchy. .. _configs-tutorial-scopes: @@ -109,11 +91,9 @@ the definition of spack's configuration scopes and their hierarchy. Configuration Scopes -------------------- -Depending on your use case, you may want to provide configuration -settings common to everyone on your team, or you may want to set -default behaviors specific to a single user account. Spack provides -six configuration *scopes* to handle this customization. These scopes, -in order of decreasing priority, are: +Depending on your use case, you may want to provide configuration settings common to everyone on your team, or you may want to set default behaviors specific to a single user account. +Spack provides six configuration *scopes* to handle this customization. +These scopes, in order of decreasing priority, are: ============ =================================================== Scope Directory @@ -127,40 +107,29 @@ System ``/etc/spack/`` Defaults ``$SPACK_ROOT/etc/spack/defaults/`` ============ =================================================== -Spack's default configuration settings reside in -``$SPACK_ROOT/etc/spack/defaults``. These are useful for reference, -but should never be directly edited. To override these settings, -create new configuration files in any of the higher-priority -configuration scopes. +Spack's default configuration settings reside in ``$SPACK_ROOT/etc/spack/defaults``. +These are useful for reference, but should never be directly edited. +To override these settings, create new configuration files in any of the higher-priority configuration scopes. -A particular cluster may have multiple Spack installations associated -with different projects. To provide settings common to all Spack -installations, put your configuration files in ``/etc/spack``. -To provide settings specific to a particular Spack installation, -you can use the ``$SPACK_ROOT/etc/spack`` directory. +A particular cluster may have multiple Spack installations associated with different projects. +To provide settings common to all Spack installations, put your configuration files in ``/etc/spack``. +To provide settings specific to a particular Spack installation, you can use the ``$SPACK_ROOT/etc/spack`` directory. -For settings specific to a particular user, you will want to add -configuration files to the ``~/.spack`` directory. When Spack first -checked for compilers on your system, you may have noticed that it -placed your compiler configuration in this directory. +For settings specific to a particular user, you will want to add configuration files to the ``~/.spack`` directory. +When Spack first checked for compilers on your system, you may have noticed that it placed your compiler configuration in this directory. -Configuration settings can also be placed in a custom location, -which is then specified on the command line via ``--config-scope``. -An example use case is managing two sets of configurations, one for -development and another for production preferences. +Configuration settings can also be placed in a custom location, which is then specified on the command line via ``--config-scope``. +An example use case is managing two sets of configurations, one for development and another for production preferences. -Settings specified on the command line have precedence over all -other configuration scopes. +Settings specified on the command line have precedence over all other configuration scopes. ^^^^^^^^^^^^^^^^^^^^^^^^ Platform-specific scopes ^^^^^^^^^^^^^^^^^^^^^^^^ -Some facilities manage multiple platforms from a single shared -file system. In order to handle this, each of the configuration -scopes listed above has two *sub-scopes*: platform-specific and -platform-independent. For example, compiler settings can be stored -in the following locations: +Some facilities manage multiple platforms from a single shared file system. +In order to handle this, each of the configuration scopes listed above has two *sub-scopes*: platform-specific and platform-independent. +For example, compiler settings can be stored in the following locations: #. ``$ENVIRONMENT_ROOT/spack.yaml`` #. ``~/.spack//compilers.yaml`` @@ -172,29 +141,21 @@ in the following locations: #. ``$SPACK_ROOT/etc/defaults//compilers.yaml`` #. ``$SPACK_ROOT/etc/defaults/compilers.yaml`` -These files are listed in decreasing order of precedence, so files in -``~/.spack/`` will override settings in ``~/.spack``. +These files are listed in decreasing order of precedence, so files in ``~/.spack/`` will override settings in ``~/.spack``. ----------- YAML Format ----------- -Spack configurations are nested YAML dictionaries with a specified -schema. The configuration is organized into sections based on theme -(e.g., a 'compilers' section) and the highest-level keys of the dictionary -specify the section. Spack generally maintains a separate file for -each section, although environments keep them together (in -``spack.yaml``). - -When Spack checks its configuration, -the configuration scopes are updated as dictionaries in increasing -order of precedence, allowing higher precedence files to override -lower. YAML dictionaries use a colon ":" to specify key-value -pairs. Spack extends YAML syntax slightly to allow a double-colon -"::" to specify a key-value pair. When a double-colon is used, -instead of adding that section, Spack replaces what was in that -section with the new value. For example, consider a user's compilers -configuration file as follows: +Spack configurations are nested YAML dictionaries with a specified schema. +The configuration is organized into sections based on theme (e.g., a 'compilers' section) and the highest-level keys of the dictionary specify the section. +Spack generally maintains a separate file for each section, although environments keep them together (in ``spack.yaml``). + +When Spack checks its configuration, the configuration scopes are updated as dictionaries in increasing order of precedence, allowing higher precedence files to override lower. +YAML dictionaries use a colon ":" to specify key-value pairs. +Spack extends YAML syntax slightly to allow a double-colon "::" to specify a key-value pair. +When a double-colon is used, instead of adding that section, Spack replaces what was in that section with the new value. +For example, consider a user's compilers configuration file as follows: .. code-block:: yaml @@ -214,18 +175,11 @@ configuration file as follows: extra_rpaths: [] -This ensures that no other compilers are used, as the user configuration -scope is the last scope searched and the ``compilers::`` line replaces -information from all previous configuration files. If the same -configuration file had a single colon instead of the double colon, it -would add the GCC version 11.3.0 compiler to whatever other compilers -were listed in other configuration files. +This ensures that no other compilers are used, as the user configuration scope is the last scope searched and the ``compilers::`` line replaces information from all previous configuration files. +If the same configuration file had a single colon instead of the double colon, it would add the GCC version 11.3.0 compiler to whatever other compilers were listed in other configuration files. -A configuration section appears nearly the same when managed in an -environment's ``spack.yaml`` file except that the section is nested 1 -level underneath the top-level 'spack' key. For example the above -``compilers.yaml`` could be incorporated into an environment's -``spack.yaml`` like so: +A configuration section appears nearly the same when managed in an environment's ``spack.yaml`` file except that the section is nested 1 level underneath the top-level 'spack' key. +For example the above ``compilers.yaml`` could be incorporated into an environment's ``spack.yaml`` like so: .. code-block:: yaml @@ -254,13 +208,10 @@ level underneath the top-level 'spack' key. For example the above Compiler Configuration ---------------------- -For most tasks, we can use Spack with the compilers auto-detected the -first time Spack runs on a system. As discussed in the basic -installation tutorial, we can also tell Spack where compilers are -located using the ``spack compiler add`` command. However, in some -circumstances we want even more fine-grained control over the -compilers available. This section will teach you how to exercise that -control using the compilers configuration file. +For most tasks, we can use Spack with the compilers auto-detected the first time Spack runs on a system. +As discussed in the basic installation tutorial, we can also tell Spack where compilers are located using the ``spack compiler add`` command. +However, in some circumstances we want even more fine-grained control over the compilers available. +This section will teach you how to exercise that control using the compilers configuration file. We will start by opening the compilers configuration file: @@ -269,9 +220,7 @@ We will start by opening the compilers configuration file: $ spack config edit compilers -We start with no active environment, so this will open a -``compilers.yaml`` file for editing (you can also do this with an -active environment): +We start with no active environment, so this will open a ``compilers.yaml`` file for editing (you can also do this with an active environment): .. code-block:: yaml @@ -316,11 +265,9 @@ active environment): environment: {} extra_rpaths: [] -This specifies two versions of the GCC compiler and one version of the -Clang compiler with no Flang compiler. Now suppose we have a code that -we want to compile with the Clang compiler for C/C++ code, but with -gfortran for Fortran components. We can do this by adding another entry -to the ``compilers.yaml`` file: +This specifies two versions of the GCC compiler and one version of the Clang compiler with no Flang compiler. +Now suppose we have a code that we want to compile with the Clang compiler for C/C++ code, but with gfortran for Fortran components. +We can do this by adding another entry to the ``compilers.yaml`` file: .. code-block:: yaml :emphasize-lines: 2,6-7 @@ -341,20 +288,16 @@ to the ``compilers.yaml`` file: Let's talk about the sections of this compiler entry that we've changed. -The biggest change we've made is to the ``paths`` section. This lists -the paths to the compilers to use for each language/specification. -In this case, we point to the Clang compiler for C/C++ and the gfortran -compiler for both specifications of Fortran. We've also changed the -``spec`` entry for this compiler. The ``spec`` entry is effectively the -name of the compiler for Spack. It consists of a name and a version -number, separated by the ``@`` sigil. The name must be one of the supported -compiler names in Spack (aocc, apple-clang, arm, cce, clang, dpcpp, fj, -gcc, intel, msvc, nag, nvhpc, oneapi, pgi, rocmcc, xl, xl_r). -The version number can be an arbitrary string of alphanumeric characters, -as well as ``-``, ``.``, and ``_``. The ``target`` and ``operating_system`` -sections we leave unchanged. These sections specify when Spack can use -different compilers, and are primarily useful for configuration files that -will be used across multiple systems. +The biggest change we've made is to the ``paths`` section. +This lists the paths to the compilers to use for each language/specification. +In this case, we point to the Clang compiler for C/C++ and the gfortran compiler for both specifications of Fortran. +We've also changed the ``spec`` entry for this compiler. +The ``spec`` entry is effectively the name of the compiler for Spack. +It consists of a name and a version number, separated by the ``@`` sigil. +The name must be one of the supported compiler names in Spack (aocc, apple-clang, arm, cce, clang, dpcpp, fj, gcc, intel, msvc, nag, nvhpc, oneapi, pgi, rocmcc, xl, xl_r). +The version number can be an arbitrary string of alphanumeric characters, as well as ``-``, ``.``, and ``_``. +The ``target`` and ``operating_system`` sections we leave unchanged. +These sections specify when Spack can use different compilers, and are primarily useful for configuration files that will be used across multiple systems. We can verify that our new compiler works by invoking it now: @@ -364,9 +307,8 @@ We can verify that our new compiler works by invoking it now: ... -This new compiler also works on Fortran codes. We'll show it by -compiling a small package using as a build dependency ``cmake%gcc@11.4.0`` -since it is already available in our binary cache: +This new compiler also works on Fortran codes. +We'll show it by compiling a small package using as a build dependency ``cmake%gcc@11.4.0`` since it is already available in our binary cache: .. code-block:: console @@ -380,13 +322,10 @@ since it is already available in our binary cache: Compiler flags ^^^^^^^^^^^^^^ -Some compilers may require specific compiler flags to work properly in -a particular computing environment. Spack provides configuration -options for setting compiler flags every time a specific compiler is -invoked. These flags become part of the package spec and therefore of -the build provenance. As on the command line, the flags are set -through the implicit build variables ``cflags``, ``cxxflags``, ``cppflags``, -``fflags``, ``ldflags``, and ``ldlibs``. +Some compilers may require specific compiler flags to work properly in a particular computing environment. +Spack provides configuration options for setting compiler flags every time a specific compiler is invoked. +These flags become part of the package spec and therefore of the build provenance. +As on the command line, the flags are set through the implicit build variables ``cflags``, ``cxxflags``, ``cppflags``, ``fflags``, ``ldflags``, and ``ldlibs``. Let's open our compilers configuration file again and add a compiler flag: @@ -409,8 +348,7 @@ Let's open our compilers configuration file again and add a compiler flag: extra_rpaths: [] -We can test this out using the ``spack spec`` command to show how the -spec is concretized: +We can test this out using the ``spack spec`` command to show how the spec is concretized: .. literalinclude:: outputs/config/0.compiler_flags.out :language: console @@ -422,11 +360,9 @@ We can see that ``cppflags="-g"`` has been added to every node in the DAG. Advanced compiler configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are four fields of the compiler configuration entry that we -have not yet talked about. +There are four fields of the compiler configuration entry that we have not yet talked about. -The ``target`` field of the compiler defines the cpu architecture **family** -that the compiler supports. +The ``target`` field of the compiler defines the cpu architecture **family** that the compiler supports. .. code-block:: yaml @@ -435,12 +371,8 @@ that the compiler supports. target: ppc64le ... -The ``modules`` field of the compiler was originally designed to -support older Cray systems, but can be useful on any system that has -compilers that are only usable when a particular module is loaded. Any -modules in the ``modules`` field of the compiler configuration will be -loaded as part of the build environment for packages using that -compiler: +The ``modules`` field of the compiler was originally designed to support older Cray systems, but can be useful on any system that has compilers that are only usable when a particular module is loaded. +Any modules in the ``modules`` field of the compiler configuration will be loaded as part of the build environment for packages using that compiler: .. code-block:: yaml @@ -451,11 +383,8 @@ compiler: - gcc/5.3.0 ... -The ``environment`` field of the compiler configuration is used for -compilers that require environment variables to be set during build -time. For example, if your Intel compiler suite requires the -``INTEL_LICENSE_FILE`` environment variable to point to the proper -license server, you can set this in ``compilers.yaml`` as follows: +The ``environment`` field of the compiler configuration is used for compilers that require environment variables to be set during build time. +For example, if your Intel compiler suite requires the ``INTEL_LICENSE_FILE`` environment variable to point to the proper license server, you can set this in ``compilers.yaml`` as follows: .. code-block:: yaml @@ -467,21 +396,14 @@ license server, you can set this in ``compilers.yaml`` as follows: ... -In addition to ``set``, ``environment`` also supports ``unset``, -``prepend_path``, and ``append_path``. +In addition to ``set``, ``environment`` also supports ``unset``, ``prepend_path``, and ``append_path``. -The ``extra_rpaths`` field of the compiler configuration is used for -compilers that do not rpath all of their dependencies by -default. Since compilers are often installed externally to Spack, -Spack is unable to manage compiler dependencies and enforce -rpath usage. This can lead to packages not finding link dependencies -imposed by the compiler properly. For compilers that impose link -dependencies on the resulting executables that are not rpath'ed into -the executable automatically, the ``extra_rpaths`` field of the compiler -configuration tells Spack which dependencies to rpath into every -executable created by that compiler. The executables will then be able -to find the link dependencies imposed by the compiler. As an example, -this field can be set by: +The ``extra_rpaths`` field of the compiler configuration is used for compilers that do not rpath all of their dependencies by default. +Since compilers are often installed externally to Spack, Spack is unable to manage compiler dependencies and enforce rpath usage. +This can lead to packages not finding link dependencies imposed by the compiler properly. +For compilers that impose link dependencies on the resulting executables that are not rpath'ed into the executable automatically, the ``extra_rpaths`` field of the compiler configuration tells Spack which dependencies to rpath into every executable created by that compiler. +The executables will then be able to find the link dependencies imposed by the compiler. +As an example, this field can be set by: .. code-block:: yaml @@ -498,9 +420,8 @@ this field can be set by: Configuring Package Preferences ------------------------------- -Package preferences in Spack are managed through the ``packages`` -configuration section. First, we will look at the default ``packages.yaml`` -file. +Package preferences in Spack are managed through the ``packages`` configuration section. +First, we will look at the default ``packages.yaml`` file. .. code-block:: console @@ -512,20 +433,17 @@ file. :emphasize-lines: 18,45 -This sets the default preferences for compilers and for providers of -virtual packages. To illustrate how this works, suppose we want to -change the preferences to prefer the Clang compiler and to prefer -MPICH over OpenMPI. Currently, we prefer GCC and OpenMPI. +This sets the default preferences for compilers and for providers of virtual packages. +To illustrate how this works, suppose we want to change the preferences to prefer the Clang compiler and to prefer MPICH over OpenMPI. +Currently, we prefer GCC and OpenMPI. .. literalinclude:: outputs/config/0.prefs.out :language: console :emphasize-lines: 16 -Let's override these default preferences in an environment. When you -have an activated environment, you can edit the associated -configuration with ``spack config edit`` (you don't have to provide -a section name): +Let's override these default preferences in an environment. +When you have an activated environment, you can edit the associated configuration with ``spack config edit`` (you don't have to provide a section name): .. code-block:: console @@ -556,8 +474,7 @@ a section name): mpi: [mpich, openmpi] -Because of the configuration scoping we discussed earlier, this -overrides the default settings just for these two items. +Because of the configuration scoping we discussed earlier, this overrides the default settings just for these two items. .. literalinclude:: outputs/config/1.prefs.out :language: console @@ -568,11 +485,9 @@ overrides the default settings just for these two items. Variant preferences ^^^^^^^^^^^^^^^^^^^ -As we've seen throughout this tutorial, HDF5 builds with MPI enabled by -default in Spack. If we were working on a project that would routinely -need serial HDF5, that might get annoying quickly, having to type -``hdf5~mpi`` all the time. Instead, we'll update our preferences for -HDF5. +As we've seen throughout this tutorial, HDF5 builds with MPI enabled by default in Spack. +If we were working on a project that would routinely need serial HDF5, that might get annoying quickly, having to type ``hdf5~mpi`` all the time. +Instead, we'll update our preferences for HDF5. .. code-block:: yaml :emphasize-lines: 9-10 @@ -596,22 +511,18 @@ Now hdf5 will concretize without an MPI dependency by default. :emphasize-lines: 8 -In general, every attribute that we can set for all packages we can -set separately for an individual package. +In general, every attribute that we can set for all packages we can set separately for an individual package. ^^^^^^^^^^^^^^^^^ External packages ^^^^^^^^^^^^^^^^^ -The packages configuration file also controls when Spack will build -against an externally installed package. Spack has a -``spack external find`` command that can automatically discover and -register externally installed packages. This works for many common -build dependencies, but it's also important to know how to do this -manually for packages that Spack cannot yet detect. +The packages configuration file also controls when Spack will build against an externally installed package. +Spack has a ``spack external find`` command that can automatically discover and register externally installed packages. +This works for many common build dependencies, but it's also important to know how to do this manually for packages that Spack cannot yet detect. -On these systems we have a pre-installed curl. Let's tell Spack about -this package and where it can be found: +On these systems we have a pre-installed curl. +Let's tell Spack about this package and where it can be found: .. code-block:: yaml :emphasize-lines: 11-14 @@ -634,25 +545,23 @@ this package and where it can be found: Here, we've told Spack that Curl 7.81.0 is installed on our system. We've also told it the installation prefix where Curl can be found. -We don't know exactly which variants it was built with, but that's -okay. +We don't know exactly which variants it was built with, but that's okay. .. literalinclude:: outputs/config/0.externals.out :language: console -You'll notice that Spack is now using the external Curl installation, -but the compiler used to build Curl is now overriding our compiler -preference of clang. If we explicitly specify Clang: +You'll notice that Spack is now using the external Curl installation, but the compiler used to build Curl is now overriding our compiler preference of clang. +If we explicitly specify Clang: .. literalinclude:: outputs/config/1.externals.out :language: console Spack concretizes to both HDF5 and Curl being built with Clang. -This has a side-effect of rebuilding Curl. If we want to force -Spack to use the system Curl, we have two choices. We can either -specify it on the command line, or we can tell Spack that it's -not allowed to build its own Curl. We'll go with the latter. +This has a side-effect of rebuilding Curl. +If we want to force Spack to use the system Curl, we have two choices. +We can either specify it on the command line, or we can tell Spack that it's not allowed to build its own Curl. +We'll go with the latter. .. code-block:: yaml :emphasize-lines: 15 @@ -680,22 +589,18 @@ Now Spack will be forced to choose the external Curl. :language: console -This gets slightly more complicated with virtual dependencies. Suppose -we don't want to build our own MPI, but we now want a parallel version -of HDF5. Well, fortunately we have MPICH installed on these systems. +This gets slightly more complicated with virtual dependencies. +Suppose we don't want to build our own MPI, but we now want a parallel version of HDF5. +Well, fortunately we have MPICH installed on these systems. -Instead of manually configuring an external for MPICH like we did for -Curl we will use the ``spack external find`` command. For packages -that support this option, this is a useful way to avoid typos and get -more accurate external specs. +Instead of manually configuring an external for MPICH like we did for Curl we will use the ``spack external find`` command. +For packages that support this option, this is a useful way to avoid typos and get more accurate external specs. .. literalinclude:: outputs/config/0.external_find.out :language: console -To express that we don't want any other MPI installed we can use the -virtual ``mpi`` package as a key. While we're editing the -``spack.yaml`` file, make sure to configure HDF5 to be able to build -with MPI again: +To express that we don't want any other MPI installed we can use the virtual ``mpi`` package as a key. +While we're editing the ``spack.yaml`` file, make sure to configure HDF5 to be able to build with MPI again: .. code-block:: yaml :emphasize-lines: 14-19 @@ -720,16 +625,15 @@ with MPI again: mpi: buildable: false -Now that we have configured Spack not to build any possible provider -for MPI, we can try again. +Now that we have configured Spack not to build any possible provider for MPI, we can try again. .. literalinclude:: outputs/config/3.externals.out :language: console :emphasize-lines: 15 -Notice that we still haven't build ``hdf5`` with our external -``mpich``. The concretizer has instead turned off ``mpi`` support in -``hdf5``. To debug this, we will force Spack to use ``hdf5+mpi``. +Notice that we still haven't build ``hdf5`` with our external ``mpich``. +The concretizer has instead turned off ``mpi`` support in ``hdf5``. +To debug this, we will force Spack to use ``hdf5+mpi``. .. code-block:: console @@ -743,31 +647,25 @@ Notice that we still haven't build ``hdf5`` with our external required because conflict is triggered when +mpi required because hdf5%clang+mpi requested from CLI -In this case, we cannot use the external mpich. The version is -incompatible with ``hdf5``. At this point, the best option is to give -up and let Spack build ``mpi`` for us. The alternative is to try to -find a version of ``hdf5`` which doesn't have this conflict. +In this case, we cannot use the external mpich. +The version is incompatible with ``hdf5``. +At this point, the best option is to give up and let Spack build ``mpi`` for us. +The alternative is to try to find a version of ``hdf5`` which doesn't have this conflict. -By configuring most of our package preferences in ``packages.yaml``, -we can cut down on the amount of work we need to do when specifying -a spec on the command line. In addition to compiler and variant -preferences, we can specify version preferences as well. Except for -specifying dependencies via ``^``, anything that you can specify on the -command line can be specified in ``packages.yaml`` with the exact -same spec syntax. +By configuring most of our package preferences in ``packages.yaml``, we can cut down on the amount of work we need to do when specifying a spec on the command line. +In addition to compiler and variant preferences, we can specify version preferences as well. +Except for specifying dependencies via ``^``, anything that you can specify on the command line can be specified in ``packages.yaml`` with the exact same spec syntax. ^^^^^^^^^^^^^^^^^^^^^^^^ Installation permissions ^^^^^^^^^^^^^^^^^^^^^^^^ -The ``packages`` configuration also controls the default permissions -to use when installing a package. You'll notice that by default, -the installation prefix will be world-readable but only user-writable. +The ``packages`` configuration also controls the default permissions to use when installing a package. +You'll notice that by default, the installation prefix will be world-readable but only user-writable. Let's say we need to install ``converge``, a licensed software package. -Since a specific research group, ``fluid_dynamics``, pays for this -license, we want to ensure that only members of this group can access -the software. We can do this like so: +Since a specific research group, ``fluid_dynamics``, pays for this license, we want to ensure that only members of this group can access the software. +We can do this like so: .. code-block:: yaml @@ -778,11 +676,9 @@ the software. We can do this like so: group: fluid_dynamics -Now, only members of the ``fluid_dynamics`` group can use any -``converge`` installations. +Now, only members of the ``fluid_dynamics`` group can use any ``converge`` installations. -At this point we want to discard the configuration changes we made -in this tutorial section, so we can deactivate the environment: +At this point we want to discard the configuration changes we made in this tutorial section, so we can deactivate the environment: .. code-block:: console @@ -800,10 +696,9 @@ in this tutorial section, so we can deactivate the environment: High-level Config ----------------- -In addition to compiler and package settings, Spack allows customization -of several high-level settings. These settings are managed in the ``config`` -section (in ``config.yaml`` when stored as an individual file outside of -an environment). You can see the default settings by running: +In addition to compiler and package settings, Spack allows customization of several high-level settings. +These settings are managed in the ``config`` section (in ``config.yaml`` when stored as an individual file outside of an environment). +You can see the default settings by running: .. code-block:: console @@ -815,11 +710,9 @@ an environment). You can see the default settings by running: As you can see, many of the directories Spack uses can be customized. -For example, you can tell Spack to install packages to a prefix -outside of the ``$SPACK_ROOT`` hierarchy. Module files can be -written to a central location if you are using multiple Spack -instances. If you have a fast scratch file system, you can run builds -from this file system with the following ``config.yaml``: +For example, you can tell Spack to install packages to a prefix outside of the ``$SPACK_ROOT`` hierarchy. +Module files can be written to a central location if you are using multiple Spack instances. +If you have a fast scratch file system, you can run builds from this file system with the following ``config.yaml``: .. code-block:: yaml @@ -838,9 +731,7 @@ from this file system with the following ``config.yaml``: `Basic Settings `_ for details. -On systems with compilers that absolutely *require* environment variables -like ``LD_LIBRARY_PATH``, it is possible to prevent Spack from cleaning -the build environment with the ``dirty`` setting: +On systems with compilers that absolutely *require* environment variables like ``LD_LIBRARY_PATH``, it is possible to prevent Spack from cleaning the build environment with the ``dirty`` setting: .. code-block:: yaml @@ -848,14 +739,11 @@ the build environment with the ``dirty`` setting: dirty: true -However, this is strongly discouraged, as it can pull unwanted libraries -into the build. +However, this is strongly discouraged, as it can pull unwanted libraries into the build. -One last setting that may be of interest to many users is the ability -to customize the parallelism of Spack builds. By default, Spack -installs all packages in parallel with the number of jobs equal to the -number of cores on the node (up to a maximum of 16). For example, on a -node with 16 cores, this will look like: +One last setting that may be of interest to many users is the ability to customize the parallelism of Spack builds. +By default, Spack installs all packages in parallel with the number of jobs equal to the number of cores on the node (up to a maximum of 16). +For example, on a node with 16 cores, this will look like: .. code-block:: console @@ -871,11 +759,10 @@ node with 16 cores, this will look like: [+] /home/user/spack/opt/spack/linux-ubuntu22.04-x86_64/gcc-11.3.0/zlib-1.2.12-fntvsj6xevbz5gyq7kfa4xg7oxnaolxs -As you can see, we are building with all 16 cores on the node. If you are -on a shared login node, this can slow down the system for other users. If -you have a strict ulimit or restriction on the number of available licenses, -you may not be able to build at all with this many cores. To limit the -number of cores our build uses, set ``build_jobs`` like so: +As you can see, we are building with all 16 cores on the node. +If you are on a shared login node, this can slow down the system for other users. +If you have a strict ulimit or restriction on the number of available licenses, you may not be able to build at all with this many cores. +To limit the number of cores our build uses, set ``build_jobs`` like so: .. code-block:: console @@ -904,11 +791,9 @@ If we uninstall and reinstall zlib-ng, we see that it now uses only 2 cores: [+] /home/user/spack/opt/spack/linux-ubuntu22.04... -Obviously, if you want to build everything in serial for whatever reason, -you would set ``build_jobs`` to 1. +Obviously, if you want to build everything in serial for whatever reason, you would set ``build_jobs`` to 1. -Last we'll unset ``concretizer:reuse:false`` since we'll want to -enable concretizer reuse for the rest of this tutorial. +Last we'll unset ``concretizer:reuse:false`` since we'll want to enable concretizer reuse for the rest of this tutorial. .. code-block:: yaml @@ -922,13 +807,9 @@ enable concretizer reuse for the rest of this tutorial. Conclusion ---------- -In this tutorial, we covered basic Spack configuration using ``compilers.yaml``, -``packages.yaml``, and ``config.yaml``. Spack has many more configuration files, -including ``modules.yaml``, which will be covered in the :ref:`modules-tutorial`. -For more detailed documentation on Spack's many configuration settings, see -`the configuration section `_ -of Spack's main documentation. +In this tutorial, we covered basic Spack configuration using ``compilers.yaml``, ``packages.yaml``, and ``config.yaml``. +Spack has many more configuration files, including ``modules.yaml``, which will be covered in the :ref:`modules-tutorial`. +For more detailed documentation on Spack's many configuration settings, see `the configuration section `_ of Spack's main documentation. -For examples of how other sites configure Spack, see -https://github.com/spack/spack-configs. If you use Spack at your site -and want to share your config files, feel free to submit a pull request! +For examples of how other sites configure Spack, see https://github.com/spack/spack-configs. +If you use Spack at your site and want to share your config files, feel free to submit a pull request! diff --git a/tutorial_developer_workflows.rst b/tutorial_developer_workflows.rst index 9cecdcc68e..5611f3cc9c 100644 --- a/tutorial_developer_workflows.rst +++ b/tutorial_developer_workflows.rst @@ -11,42 +11,31 @@ Developer Workflows Tutorial ============================ -This tutorial will guide you through the process of using the ``spack -develop`` command to develop software from local source code within a -spack environment. With this command spack will manage your -dependencies while you focus on testing changes to your library and/or -application. +This tutorial will guide you through the process of using the ``spack develop`` command to develop software from local source code within a spack environment. +With this command spack will manage your dependencies while you focus on testing changes to your library and/or application. ----------------------------- Installing from local source ----------------------------- -The ``spack install`` command, as you know, fetches source code from a -mirror or the internet before building and installing your package. As -developers, we want to build from local source, which we will -constantly change, build, and test. - -Let's imagine for a second we're working on ``scr``. ``scr`` is a -library used to implement scalable checkpointing in application -codes. It supports writing/reading checkpoints quickly and efficiently -using MPI and high-bandwidth file I/O. We'd like to test changes to -scr within an actual application so we'll test with ``macsio``, a -proxy application written to mimic typical HPC I/O workloads. We've -chosen ``scr`` and ``macsio`` because together they are quick to -build. - -We'll start by making an environment for our development. We need to -build ``macsio`` with ``scr`` support, and we'd like everything to be -built without fortran support for the time being. Let's set up that -development workflow. +The ``spack install`` command, as you know, fetches source code from a mirror or the internet before building and installing your package. +As developers, we want to build from local source, which we will constantly change, build, and test. + +Let's imagine for a second we're working on ``scr``. ``scr`` is a library used to implement scalable checkpointing in application codes. +It supports writing/reading checkpoints quickly and efficiently using MPI and high-bandwidth file I/O. +We'd like to test changes to scr within an actual application so we'll test with ``macsio``, a proxy application written to mimic typical HPC I/O workloads. +We've chosen ``scr`` and ``macsio`` because together they are quick to build. + +We'll start by making an environment for our development. +We need to build ``macsio`` with ``scr`` support, and we'd like everything to be built without fortran support for the time being. +Let's set up that development workflow. .. literalinclude:: outputs/dev/setup-scr.out :language: console -Before we do any work, we verify that this all builds. Spack ends up -building the entire development tree below, and links everything -together for you. +Before we do any work, we verify that this all builds. +Spack ends up building the entire development tree below, and links everything together for you. .. graphviz:: @@ -149,22 +138,18 @@ Now we are ready to begin work on the actual application. Development iteration cycles ----------------------------- -Let's assume that scr has a bug, and we'd like to patch scr to find -out what the problem is. First, we tell spack that we'd like to check -out the version of scr that we want to work on. In this case, it will -be the 3.1.0 release that we want to write a patch for: +Let's assume that scr has a bug, and we'd like to patch scr to find out what the problem is. +First, we tell spack that we'd like to check out the version of scr that we want to work on. +In this case, it will be the 3.1.0 release that we want to write a patch for: .. literalinclude:: outputs/dev/develop-1.out :language: console -The spack develop command marks the package as being a "development" -package in the spack.yaml. This adds a special ``dev_path=`` attribute -to the spec for the package, so spack remembers where the source code -for this package is located. The develop command also downloads/checks -out the source code for the package. By default, the source code is -downloaded into a subdirectory of the environment. You can change the -location of this source directory by modifying the ``path:`` attribute -of the develop configuration in the environment. +The spack develop command marks the package as being a "development" package in the spack.yaml. +This adds a special ``dev_path=`` attribute to the spec for the package, so spack remembers where the source code for this package is located. +The develop command also downloads/checks out the source code for the package. +By default, the source code is downloaded into a subdirectory of the environment. +You can change the location of this source directory by modifying the ``path:`` attribute of the develop configuration in the environment. There are a few gotchas with the spack develop command @@ -186,54 +171,42 @@ There are a few gotchas with the spack develop command .. literalinclude:: outputs/dev/develop-conc.out :language: console -Now that we have this done, we tell spack to rebuild both ``scr`` and -``macsio`` by running ``spack install``. +Now that we have this done, we tell spack to rebuild both ``scr`` and ``macsio`` by running ``spack install``. .. literalinclude:: outputs/dev/develop-2.out :language: console -This rebuilds ``scr`` from the subdirectory we specified. If your -package uses cmake, spack will build the package in a build directory -that matches the hash for your package. From here you can change into -the appropriate directory and perform your own build/test cycles. +This rebuilds ``scr`` from the subdirectory we specified. +If your package uses cmake, spack will build the package in a build directory that matches the hash for your package. +From here you can change into the appropriate directory and perform your own build/test cycles. -Now, we can develop our code. For the sake of this demo, we're just -going to intentionally introduce an error. Let's edit a file and -remove the first semi-colon we find. +Now, we can develop our code. +For the sake of this demo, we're just going to intentionally introduce an error. +Let's edit a file and remove the first semi-colon we find. .. literalinclude:: outputs/dev/edit-1.out :language: console -Once you have a development package, ``spack install`` also works much -like "make". Since spack knows the source code directory of the -package, it checks the filetimes on the source directory to see if -we've made recent changes. If the file times are newer, it will -rebuild ``scr`` and any other package that depends on ``scr``. +Once you have a development package, ``spack install`` also works much like "make". +Since spack knows the source code directory of the package, it checks the filetimes on the source directory to see if we've made recent changes. +If the file times are newer, it will rebuild ``scr`` and any other package that depends on ``scr``. .. literalinclude:: outputs/dev/develop-3.out :language: console -Here, the build failed as expected. We can look at the output for the -build in ``scr/spack-build-out.txt`` to find out why, or we can -launch a shell directly with the appropriate environment variables to -figure out what went wrong by using ``spack build-env scr@2.0 -- -bash``. If that's too much to remember, then sourcing -``scr/spack-build-env.txt`` will also set all the appropriate -environment variables so we can diagnose the build ourselves. Now -let's fix it and rebuild directly. +Here, the build failed as expected. +We can look at the output for the build in ``scr/spack-build-out.txt`` to find out why, or we can launch a shell directly with the appropriate environment variables to figure out what went wrong by using ``spack build-env scr@2.0 -- bash``. +If that's too much to remember, then sourcing ``scr/spack-build-env.txt`` will also set all the appropriate environment variables so we can diagnose the build ourselves. +Now let's fix it and rebuild directly. .. literalinclude:: outputs/dev/develop-4.out :language: console -You'll notice here that spack rebuilt both ``scr`` and ``macsio``, as -expected. +You'll notice here that spack rebuilt both ``scr`` and ``macsio``, as expected. -Taking advantage of iterative builds with spack requires cooperation -from your build system. When spack performs a rebuild on a -development package, it reruns all the build stages for your package -without cleaning the source and build directories to a pristine -state. If your build system can take advantage of the previously -compiled object files then you'll end up with an iterative build. +Taking advantage of iterative builds with spack requires cooperation from your build system. +When spack performs a rebuild on a development package, it reruns all the build stages for your package without cleaning the source and build directories to a pristine state. +If your build system can take advantage of the previously compiled object files then you'll end up with an iterative build. - If your package just uses make, you also should get iterative builds for free when running ``spack develop``. @@ -245,26 +218,22 @@ compiled object files then you'll end up with an iterative build. ``autoreconf`` stage typically modifies the filetime of ``config.h``, which can trigger a cascade of rebuilding. -Multiple packages can also be marked as develop. If we were -co-developing ``macsio``, we could run +Multiple packages can also be marked as develop. +If we were co-developing ``macsio``, we could run .. literalinclude:: outputs/dev/develop-5.out :language: console -Using development workflows also lets us ship our whole development -process to another developer on the team. They can simply take our -spack.yaml, create a new environment, and use this to replicate our -build process. For example, we'll make another development environment -here. +Using development workflows also lets us ship our whole development process to another developer on the team. +They can simply take our spack.yaml, create a new environment, and use this to replicate our build process. +For example, we'll make another development environment here. .. literalinclude:: outputs/dev/otherdevel.out :language: console -Here, ``spack develop`` with no arguments will check out or download -the source code and place it in the appropriate places. +Here, ``spack develop`` with no arguments will check out or download the source code and place it in the appropriate places. -When we're done developing, we simply tell spack that it no longer -needs to keep a development version of the package. +When we're done developing, we simply tell spack that it no longer needs to keep a development version of the package. .. literalinclude:: outputs/dev/wrapup.out :language: console @@ -273,22 +242,17 @@ needs to keep a development version of the package. Workflow Summary ------------------- -Use the ``spack develop`` command with an environment to make a -reproducible build environment for your development workflow. Spack -will set up all the dependencies for you and link all your packages -together. Within a development environment, ``spack install`` works -similar to ``make`` in that it will check file times to rebuild the -minimum number of spack packages necessary to reflect the changes to -your build. +Use the ``spack develop`` command with an environment to make a reproducible build environment for your development workflow. +Spack will set up all the dependencies for you and link all your packages together. +Within a development environment, ``spack install`` works similar to ``make`` in that it will check file times to rebuild the minimum number of spack packages necessary to reflect the changes to your build. ------------------- Optional: Tips and Tricks ------------------- -This section will cover some additional features that are useful additions -to the core tutorial above. Many of these items are very useful to specific -projects and developers. A list of the options for the ``spack develop`` can -be viewed below: +This section will cover some additional features that are useful additions to the core tutorial above. +Many of these items are very useful to specific projects and developers. +A list of the options for the ``spack develop`` can be viewed below: .. code-block:: console @@ -297,13 +261,11 @@ be viewed below: Source Code Management ---------- -``spack develop`` allows users to manipulate the source code locations -The default behavior is to let spack manage its location and cloning operations, -but software developers often want more control over these. +``spack develop`` allows users to manipulate the source code locations The default behavior is to let spack manage its location and cloning operations, but software developers often want more control over these. The source directory can be set with the ``--path`` argument when calling ``spack develop``. -If this directory already exists then ``spack develop`` will not attempt to fetch the code -for you. This allows developers to pre-clone the software or use preferred paths as they wish. +If this directory already exists then ``spack develop`` will not attempt to fetch the code for you. +This allows developers to pre-clone the software or use preferred paths as they wish. .. code-block:: console @@ -317,16 +279,12 @@ for you. This allows developers to pre-clone the software or use preferred paths Navigation and the Build Environment ---------- -Diving into the build environment was introduced previously in the packaging section with the -``spack build-env scr -- bash`` command. This is a helpful function because it allows you -to run commands inside the build environment. In the packages section of the tutorial -this was combined with ``spack cd`` to produce a manual build outside of Spack's automated -Process. -This command is particularly useful in developer environments -- it allows developers a streamlined -workflow when iterating on a single package without the overhead of the ``spack install`` command. -The additional features of the install command are unnecessary when tightly iterating between building - and testing a particular package. For example, the workflow modifying ``scr`` that we just went through - can be simplified to: +Diving into the build environment was introduced previously in the packaging section with the ``spack build-env scr -- bash`` command. +This is a helpful function because it allows you to run commands inside the build environment. +In the packages section of the tutorial this was combined with ``spack cd`` to produce a manual build outside of Spack's automated Process. +This command is particularly useful in developer environments -- it allows developers a streamlined workflow when iterating on a single package without the overhead of the ``spack install`` command. +The additional features of the install command are unnecessary when tightly iterating between building and testing a particular package. +For example, the workflow modifying ``scr`` that we just went through can be simplified to: .. code-block:: console @@ -345,20 +303,15 @@ The additional features of the install command are unnecessary when tightly iter $ make test $ exit -Working with the build environment and along with spack navigation features -provides a nice way to iterate quickly and navigate through the hash heavy -spack directory structures. +Working with the build environment and along with spack navigation features provides a nice way to iterate quickly and navigate through the hash heavy spack directory structures. Combinatorics ------------ -The final note we will look at in this tutorial will be the power of combinatoric -development builds. There are many instances where developers want to see how -a single set of changes affects multiple builds i.e. ``+cuda`` vs ``~cuda``, -``%gcc`` vs ``%clang``, ``build_type=Release`` vs ``build_type=Debug``, etc. +The final note we will look at in this tutorial will be the power of combinatoric development builds. +There are many instances where developers want to see how a single set of changes affects multiple builds i.e. ``+cuda`` vs ``~cuda``, ``%gcc`` vs ``%clang``, ``build_type=Release`` vs ``build_type=Debug``, etc. -Developers can achieve builds of both cases from a single ``spack install`` as -long as the develop spec is generic enough to cover the packages' spec variations +Developers can achieve builds of both cases from a single ``spack install`` as long as the develop spec is generic enough to cover the packages' spec variations .. code-block:: console @@ -370,16 +323,8 @@ long as the develop spec is generic enough to cover the packages' spec variation # Inspect the graph for multiple dev_path= $ spack concretize -f -While we won't build out this example it illustrates how the ``dev_path`` for -``build_type=Release`` and ``build_type=Debug`` points to the same source code. - -Now if we want to do most of our incremental builds using the ``Release`` build -and periodically check the results using the ``Debug`` build we can combine the -workflow from the previous example: dive into the ``Release`` versions build -environment using ``spack build-env scr build_type=Release -- bash`` and -navigate with ``spack cd -b scr build_type=Release``. Note that since there -are two ``scr`` specs in the environment we must distinguish which one we -want for these commands. When we are ready to check our changes for the debug -build we can exit out of the build environment subshell, -rerun ``spack install`` to rebuild everything, and then inspect the debug build -through our method of choice. +While we won't build out this example it illustrates how the ``dev_path`` for ``build_type=Release`` and ``build_type=Debug`` points to the same source code. + +Now if we want to do most of our incremental builds using the ``Release`` build and periodically check the results using the ``Debug`` build we can combine the workflow from the previous example: dive into the ``Release`` versions build environment using ``spack build-env scr build_type=Release -- bash`` and navigate with ``spack cd -b scr build_type=Release``. +Note that since there are two ``scr`` specs in the environment we must distinguish which one we want for these commands. +When we are ready to check our changes for the debug build we can exit out of the build environment subshell, rerun ``spack install`` to rebuild everything, and then inspect the debug build through our method of choice. diff --git a/tutorial_environments.rst b/tutorial_environments.rst index bbe91c5ea4..c48d20277b 100644 --- a/tutorial_environments.rst +++ b/tutorial_environments.rst @@ -11,8 +11,7 @@ Environments Tutorial ===================== -We've covered how to install, remove, and list packages with Spack using the -commands: +We've covered how to install, remove, and list packages with Spack using the commands: * `spack install `_ to install packages; * `spack uninstall `_ to remove them; and @@ -21,17 +20,11 @@ commands: .. Customizing Spack's installation with configuration files, like `packages.yaml `_, was also discussed. -This section of the tutorial introduces **Spack Environments**, which allow you -to work with independent groups of packages separately, in a reproducible way. -In some ways, Spack environments are similar to *virtual environments* in other -systems (e.g., `Python venv `_), -but they are based around file formats (``spack.yaml`` and ``spack.lock``) that can -be shared easily and re-used by others across systems. +This section of the tutorial introduces **Spack Environments**, which allow you to work with independent groups of packages separately, in a reproducible way. +In some ways, Spack environments are similar to *virtual environments* in other systems (e.g., `Python venv `_), but they are based around file formats (``spack.yaml`` and ``spack.lock``) that can be shared easily and re-used by others across systems. -Administering properly configured software involving lots of packages -and/or varying configuration requirements (e.g., different implementations -of ``mpi``) for multiple projects and efforts can be overwhelming. Spack -environments allow you to readily: +Administering properly configured software involving lots of packages and/or varying configuration requirements (e.g., different implementations of ``mpi``) for multiple projects and efforts can be overwhelming. +Spack environments allow you to readily: * establish standard software requirements for your project(s); * set up run environments for users; @@ -40,12 +33,9 @@ environments allow you to readily: * reproduce builds (approximately or exactly) on other machines; and * much more. -This tutorial introduces the basics of creating and using environments, -then explains how to expand, configure, and build software in them. -We will start with the command line interface, then cover editing key -environment file directly. We will describe the difference between -Spack-managed and independent environments, then finish with a section -on reproducible builds. +This tutorial introduces the basics of creating and using environments, then explains how to expand, configure, and build software in them. +We will start with the command line interface, then cover editing key environment file directly. +We will describe the difference between Spack-managed and independent environments, then finish with a section on reproducible builds. ------------------- Environment Basics @@ -57,38 +47,32 @@ Let's look at the output of ``spack find`` at this point in the tutorial. :language: console -This is a complete, but cluttered list of the installed packages and -their dependencies. It contains packages built with both ``openmpi`` -and ``mpich``, as well as multiple variants of other packages, like -``hdf5`` and ``zlib-ng``. The query mechanism we learned about with -``spack find`` can help, but it would be nice if we could start from -a clean slate without losing what we've already installed. +This is a complete, but cluttered list of the installed packages and their dependencies. +It contains packages built with both ``openmpi`` and ``mpich``, as well as multiple variants of other packages, like ``hdf5`` and ``zlib-ng``. +The query mechanism we learned about with ``spack find`` can help, but it would be nice if we could start from a clean slate without losing what we've already installed. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Creating and activating environments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The ``spack env`` command can help. Let's create a new environment -called ``myproject``: +The ``spack env`` command can help. +Let's create a new environment called ``myproject``: .. literalinclude:: outputs/environments/env-create-1.out :language: console -An environment is like a virtualized Spack instance that you can -use to aggregate package installations for a project or other purpose. -It has an associated *view*, which is a single prefix where all packages -from the environment are linked. +An environment is like a virtualized Spack instance that you can use to aggregate package installations for a project or other purpose. +It has an associated *view*, which is a single prefix where all packages from the environment are linked. -You can see the environments we've created so far using the ``spack env -list`` command: +You can see the environments we've created so far using the ``spack env list`` command: .. literalinclude:: outputs/environments/env-list-1.out :language: console -Now let's **activate** our environment. You can use ``spack env activate`` -command: +Now let's **activate** our environment. +You can use ``spack env activate`` command: .. literalinclude:: outputs/environments/env-activate-1.out :language: console @@ -108,28 +92,24 @@ You can also use the shorter ``spacktivate`` alias for ``spack env activate``. can be set with the command ``shopt -s expand_aliases``. -Once you activate an environment, ``spack find`` only shows what is -in the current environment. We just created this environment, so it -does not contain any installed packages. +Once you activate an environment, ``spack find`` only shows what is in the current environment. +We just created this environment, so it does not contain any installed packages. .. literalinclude:: outputs/environments/find-env-1.out :language: console -The output from ``spack find`` is now *slightly* different. It tells -you that you're in the ``myproject`` environment, so there is no need -to panic when you see that none of the previously installed packages -are available. It also states that there are **no** *root specs*. We'll -get back to what that means later. +The output from ``spack find`` is now *slightly* different. +It tells you that you're in the ``myproject`` environment, so there is no need to panic when you see that none of the previously installed packages are available. +It also states that there are **no** *root specs*. +We'll get back to what that means later. -If you *only* want to check what environment you are in, you can use -``spack env status``: +If you *only* want to check what environment you are in, you can use ``spack env status``: .. literalinclude:: outputs/environments/env-status-1.out :language: console -If you want to leave this environment, you can use ``spack env deactivate`` -or the ``despacktivate`` alias for short. +If you want to leave this environment, you can use ``spack env deactivate`` or the ``despacktivate`` alias for short. After deactivating, we can see everything installed in this Spack instance: @@ -137,33 +117,27 @@ After deactivating, we can see everything installed in this Spack instance: :language: console -Notice that we are no longer in an environment and all our packages -are still installed. +Notice that we are no longer in an environment and all our packages are still installed. ^^^^^^^^^^^^^^^^^^^ Installing packages ^^^^^^^^^^^^^^^^^^^ -Now that we understand how creation and activation work, let's go -back to ``myproject`` and *install* a couple of packages, specifically, -``tcl`` and ``trilinos``. +Now that we understand how creation and activation work, let's go back to ``myproject`` and *install* a couple of packages, specifically, ``tcl`` and ``trilinos``. Try the usual install command first: .. literalinclude:: outputs/environments/env-fail-install-1.out :language: console -Environments are special in that you must *add* specs to them before installing. -``spack add`` allows us to do queue up several specs to be installed together. +Environments are special in that you must *add* specs to them before installing. ``spack add`` allows us to do queue up several specs to be installed together. Let's try it: .. literalinclude:: outputs/environments/env-add-1.out :language: console -Now, ``tcl`` and ``trilinos`` have been registered as **root specs** in this -environment. That is because we explicitly asked for them to -be installed, so they are the **roots** of the combined graph -of all packages we'll install. +Now, ``tcl`` and ``trilinos`` have been registered as **root specs** in this environment. +That is because we explicitly asked for them to be installed, so they are the **roots** of the combined graph of all packages we'll install. Now, let's install: @@ -171,10 +145,8 @@ Now, let's install: :language: console -We see that ``tcl`` and the dependencies of ``trilinos`` are -already installed, and that ``trilinos`` was newly installed. -We also see that the environment's view was updated -to include the new installations. +We see that ``tcl`` and the dependencies of ``trilinos`` are already installed, and that ``trilinos`` was newly installed. +We also see that the environment's view was updated to include the new installations. Now confirm the contents of the environment using ``spack find``: @@ -187,8 +159,7 @@ We can see that the roots and all their dependencies have been installed. Creating an environment incrementally ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As a short-hand, you can use the ``install --add`` flag to accomplish -the same thing in one step: +As a short-hand, you can use the ``install --add`` flag to accomplish the same thing in one step: .. code-block:: console @@ -203,16 +174,13 @@ You can also add and install specs to an environment incrementally. For example: $ spack install --add tcl $ spack install --add trilinos -If you create environments incrementally, Spack ensures that already installed -roots are not re-concretized. So, adding specs to an environment at a later point -in time will not cause existing packages to rebuild. +If you create environments incrementally, Spack ensures that already installed roots are not re-concretized. +So, adding specs to an environment at a later point in time will not cause existing packages to rebuild. -Do note however that incrementally creating an environment can give you different -package versions from an environment created all at once. We will cover this after -we've discussed different concretization strategies. +Do note however that incrementally creating an environment can give you different package versions from an environment created all at once. +We will cover this after we've discussed different concretization strategies. -Further, there are two other advantages of concretizing and installing an environment -all at once: +Further, there are two other advantages of concretizing and installing an environment all at once: * If you have a number of specs that can be installed together, adding them first and installing them together enables them to @@ -225,33 +193,24 @@ Using packages ^^^^^^^^^^^^^^ Environments provide a convenient way for using installed packages. -Running ``spack env activate`` gives you everything in the environment -on your ``PATH``. Otherwise, you would need to use `spack load -`_ -or `module load -`_ -for each package in order to set up the environment for the package (and -its dependencies). - -When you install packages into an environment, they are, by default, -linked into a single prefix, or *view*. Activating the environment -with ``spack env activate`` results in subdirectories from the view -being added to ``PATH``, ``MANPATH``, ``CMAKE_PREFIX_PATH``, -and other environment variables. This makes the environment easier to use. - -Let's try it out. We just installed ``tcl`` into our ``myproject`` -environment. ``Tcl`` includes a shell-like application called ``tclsh``. +Running ``spack env activate`` gives you everything in the environment on your ``PATH``. +Otherwise, you would need to use `spack load `_ or `module load `_ for each package in order to set up the environment for the package (and its dependencies). + +When you install packages into an environment, they are, by default, linked into a single prefix, or *view*. +Activating the environment with ``spack env activate`` results in subdirectories from the view being added to ``PATH``, ``MANPATH``, ``CMAKE_PREFIX_PATH``, and other environment variables. +This makes the environment easier to use. + +Let's try it out. +We just installed ``tcl`` into our ``myproject`` environment. ``Tcl`` includes a shell-like application called ``tclsh``. You can see the path to ``tclsh`` using ``which``: .. literalinclude:: outputs/environments/use-tcl-1.out :language: console -Notice its path includes the name of our environment *and* a ``view`` -subdirectory. +Notice its path includes the name of our environment *and* a ``view`` subdirectory. -You can now run ``tclsh`` like you would any other program that is -in your path: +You can now run ``tclsh`` like you would any other program that is in your path: .. code-block:: console @@ -264,85 +223,70 @@ in your path: Uninstalling packages ^^^^^^^^^^^^^^^^^^^^^ -We can uninstall packages from an environment without affecting -other environments. This is possible since, while Spack shares -common installations, environments only link to those installations. +We can uninstall packages from an environment without affecting other environments. +This is possible since, while Spack shares common installations, environments only link to those installations. Let's demonstrate this feature by creating another environment. -Suppose ``myproject`` requires ``trilinos`` but we have another -project that has it installed but no longer requires it. +Suppose ``myproject`` requires ``trilinos`` but we have another project that has it installed but no longer requires it. -Start by creating a ``myproject2`` environment with the installed -packages ``scr`` and ``trilinos``. +Start by creating a ``myproject2`` environment with the installed packages ``scr`` and ``trilinos``. .. literalinclude:: outputs/environments/env-create-2.out :language: console -Now we have two environments. The ``myproject`` environment has ``tcl`` -and ``trilinos`` while the ``myproject2`` environment has ``scr`` -and ``trilinos``. +Now we have two environments. +The ``myproject`` environment has ``tcl`` and ``trilinos`` while the ``myproject2`` environment has ``scr`` and ``trilinos``. -Now let's try to uninstall ``trilinos`` from ``myproject2`` and review the -contents of the environment: +Now let's try to uninstall ``trilinos`` from ``myproject2`` and review the contents of the environment: .. literalinclude:: outputs/environments/env-uninstall-1.out :language: console -We can see that ``trilinos`` won't be uninstalled because it is still referenced -in another environment managed by spack. If we want to remove it from the roots -list we need to use ``spack remove``: +We can see that ``trilinos`` won't be uninstalled because it is still referenced in another environment managed by spack. +If we want to remove it from the roots list we need to use ``spack remove``: .. literalinclude:: outputs/environments/env-remove-1.out :language: console -When the spec is first removed, we see that it is no longer a root but -is still present in the installed specs. Once we reconcretize, the -vestigial spec is removed. Now, it is no longer a root and will need -to be re-added before being installed as part of this environment. +When the spec is first removed, we see that it is no longer a root but is still present in the installed specs. +Once we reconcretize, the vestigial spec is removed. +Now, it is no longer a root and will need to be re-added before being installed as part of this environment. -We know ``trilinos`` is still needed for the ``myproject`` environment, so -let's switch back to confirm that it is still installed in that environment. +We know ``trilinos`` is still needed for the ``myproject`` environment, so let's switch back to confirm that it is still installed in that environment. .. literalinclude:: outputs/environments/env-swap-1.out :language: console -Phew! We see that ``myproject`` still has ``trilinos`` as a root -spec. Spack uses reference counting to ensure that we don't remove -``trilinos`` when it is still needed by ``myproject``. +Phew! +We see that ``myproject`` still has ``trilinos`` as a root spec. +Spack uses reference counting to ensure that we don't remove ``trilinos`` when it is still needed by ``myproject``. .. note:: Trilinos would only have been uninstalled by Spack if it were no longer needed by any environments or their package dependencies. -You can also uninstall a package and remove it from the environment -in one go with ``spack uninstall --remove trilinos``. +You can also uninstall a package and remove it from the environment in one go with ``spack uninstall --remove trilinos``. ----------------------- The ``spack.yaml`` file ----------------------- -An environment is more than just a list of root specs. It includes -*configuration* settings that affect the way Spack behaves when the -environment is activated. So far, ``myproject`` relies on configuration -defaults that can be overridden. Here we'll look at how to add specs -and ensure all the packages depending on ``mpi`` build with ``mpich``. -We can customize the selection of the ``mpi`` provider using -`concretization preferences -`_ -to change the behavior of the concretizer. +An environment is more than just a list of root specs. +It includes *configuration* settings that affect the way Spack behaves when the environment is activated. +So far, ``myproject`` relies on configuration defaults that can be overridden. +Here we'll look at how to add specs and ensure all the packages depending on ``mpi`` build with ``mpich``. +We can customize the selection of the ``mpi`` provider using `concretization preferences `_ to change the behavior of the concretizer. -Let's start by looking at the configuration of our environment using -``spack config get``: +Let's start by looking at the configuration of our environment using ``spack config get``: .. literalinclude:: outputs/environments/config-get-1.out :emphasize-lines: 8-13 -The output shows the special ``spack.yaml`` configuration file that Spack -uses to store the environment configuration. +The output shows the special ``spack.yaml`` configuration file that Spack uses to store the environment configuration. There are several important parts of this file: @@ -352,17 +296,11 @@ There are several important parts of this file: * ``concretizer:unify:``: This controls how the specs in the environment are concretized. -The ``specs`` list should look familiar; these are the specs we've been -modifying with ``spack add``. +The ``specs`` list should look familiar; these are the specs we've been modifying with ``spack add``. -``concretizer:unify:true``, the default, means that they are concretized -*together*, so that there is only one version of each package in the -environment. Other options for ``unify`` are ``false`` and ``when_possible``. -``false`` means that the specs are concretized *independently*, so that -there may be multiple versions of the same package in the environment. -``when_possible`` lies between those options. In this case, Spack will unify -as many packages in the environment, but will not fail if it cannot unify -all of them. +``concretizer:unify:true``, the default, means that they are concretized *together*, so that there is only one version of each package in the environment. +Other options for ``unify`` are ``false`` and ``when_possible``. ``false`` means that the specs are concretized *independently*, so that there may be multiple versions of the same package in the environment. ``when_possible`` lies between those options. +In this case, Spack will unify as many packages in the environment, but will not fail if it cannot unify all of them. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -374,11 +312,10 @@ Editing environment configuration Before proceeding, make sure your ``EDITOR`` environment variable is set to the path of your preferred text editor. -Let's edit ``spack.yaml`` to *require* ``mpich`` as our ``mpi`` provider -using ``spack config edit``. +Let's edit ``spack.yaml`` to *require* ``mpich`` as our ``mpi`` provider using ``spack config edit``. -You should now have the above file open in your editor. Change it -to include the ``packages:mpi:require`` entry below: +You should now have the above file open in your editor. +Change it to include the ``packages:mpi:require`` entry below: .. code-block:: yaml :emphasize-lines: 6-8 @@ -403,52 +340,39 @@ to include the ``packages:mpi:require`` entry below: more detail in the :ref:`configuration tutorial `. -We've only scratched the surface here by requiring a specific -``mpi`` provider for packages depending on ``mpi``. There are many -other customizations you can make to an environment. +We've only scratched the surface here by requiring a specific ``mpi`` provider for packages depending on ``mpi``. +There are many other customizations you can make to an environment. Refer to the links at the end of this section for more information. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Re-concretizing the environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You may need to re-install packages in the environment after making -significant changes to the configuration, such as changing virtual -providers. This can be accomplished by forcing Spack to re-concretize -the environment and re-install the specs. +You may need to re-install packages in the environment after making significant changes to the configuration, such as changing virtual providers. +This can be accomplished by forcing Spack to re-concretize the environment and re-install the specs. -For example, the packages installed in our ``myproject`` environment -are now out of sync with our new configuration since we already -installed part of the environment with ``openmpi``. Suppose we want -to install everything in ``myproject`` with ``mpich``. +For example, the packages installed in our ``myproject`` environment are now out of sync with our new configuration since we already installed part of the environment with ``openmpi``. +Suppose we want to install everything in ``myproject`` with ``mpich``. -Let's run ``spack concretize --force`` (or ``-f`` in short) to make -Spack re-concretize all the environment's specs: +Let's run ``spack concretize --force`` (or ``-f`` in short) to make Spack re-concretize all the environment's specs: .. literalinclude:: outputs/environments/concretize-f-1.out :language: console -All the specs are now concrete **and** ready to be installed with -``mpich`` as the MPI implementation. +All the specs are now concrete **and** ready to be installed with ``mpich`` as the MPI implementation. -Re-concretization is sometimes also necessary when creating an -environment *incrementally* with unification enabled. Spack makes -sure that already concretized specs in the environment are not -modified when adding something new. +Re-concretization is sometimes also necessary when creating an environment *incrementally* with unification enabled. +Spack makes sure that already concretized specs in the environment are not modified when adding something new. -Adding and installing specs one by one leads to greedy -concretization. When you first install ``python`` in an environment, -Spack will pick a recent version for it. If you then add ``py-numpy``, -it may be in conflict with the ``python`` version already installed, -and fail to concretize: +Adding and installing specs one by one leads to greedy concretization. +When you first install ``python`` in an environment, Spack will pick a recent version for it. +If you then add ``py-numpy``, it may be in conflict with the ``python`` version already installed, and fail to concretize: .. literalinclude:: outputs/environments/incremental-1.out :language: console -The solution is to re-concretize the environment as a whole, -which causes ``python`` to downgrade to a version compatible -with ``py-numpy``: +The solution is to re-concretize the environment as a whole, which causes ``python`` to downgrade to a version compatible with ``py-numpy``: .. literalinclude:: outputs/environments/incremental-2.out :language: console @@ -457,36 +381,29 @@ with ``py-numpy``: Building in environments ------------------------ -Activated environments allow you to invoke any programs installed -in them as if they were installed on the system. In this section -we will take advantage of that feature. +Activated environments allow you to invoke any programs installed in them as if they were installed on the system. +In this section we will take advantage of that feature. -Suppose you want to compile some MPI programs. We have an MPI -implementation installed in our ``myproject2`` environment, so -``mpicc`` is available in our path. We can confirm this using -``which``: +Suppose you want to compile some MPI programs. +We have an MPI implementation installed in our ``myproject2`` environment, so ``mpicc`` is available in our path. +We can confirm this using ``which``: .. literalinclude:: outputs/environments/show-mpicc-1.out :language: console -As mentioned before, activating the environment sets a number of -environment variables. That includes variables like ``PATH``, -``MANPATH``, and ``CMAKE_PREFIX_PATH``, which allows you to -easily find package executables and libraries installed in the environment. +As mentioned before, activating the environment sets a number of environment variables. +That includes variables like ``PATH``, ``MANPATH``, and ``CMAKE_PREFIX_PATH``, which allows you to easily find package executables and libraries installed in the environment. -Let's look specifically at path-related environment variables using -``env | grep PATH``: +Let's look specifically at path-related environment variables using ``env | grep PATH``: .. literalinclude:: outputs/environments/show-paths-1.out :language: console -We can demonstrate use of these environment settings by building a -really simple MPI program. +We can demonstrate use of these environment settings by building a really simple MPI program. -Let's create a program called ``mpi-hello.c`` that contains the following -code: +Let's create a program called ``mpi-hello.c`` that contains the following code: .. code-block:: c @@ -510,8 +427,7 @@ code: } This program includes headers from ``mpi`` and ``zlib``. -It also prints out a message from each MPI rank and the -version of ``zlib``. +It also prints out a message from each MPI rank and the version of ``zlib``. Let's build and run our program: @@ -519,13 +435,10 @@ Let's build and run our program: :language: console -Notice that we only needed to pass the include path to the -compiler. -We also see that ``Hello world`` is output for each of the ranks -and the version of ``zlib`` used to build the program is printed. +Notice that we only needed to pass the include path to the compiler. +We also see that ``Hello world`` is output for each of the ranks and the version of ``zlib`` used to build the program is printed. -We can confirm the version of ``zlib`` used to build the program -is in our environment using ``spack find``: +We can confirm the version of ``zlib`` used to build the program is in our environment using ``spack find``: .. literalinclude:: outputs/environments/myproject-zlib-ng-1.out :language: console @@ -536,24 +449,19 @@ Note that the reported version *does* match that of our installation. Reproducing builds ------------------ -Spack environments provide users with *virtual environments* -similar to `Python venv `_ -and `Conda environments -`_). The goal is to ensure packages in one environment -are kept separate from those of another. These environments can -be managed by Spack or independent. In either case, their environment -files can be used to reproduce builds by other users and on other machines. +Spack environments provide users with *virtual environments* similar to `Python venv `_ and `Conda environments `_). +The goal is to ensure packages in one environment are kept separate from those of another. +These environments can be managed by Spack or independent. +In either case, their environment files can be used to reproduce builds by other users and on other machines. Since those files are key to reproducing builds, let's start with them. ^^^^^^^^^^^^^^^^^ Environment files ^^^^^^^^^^^^^^^^^ -There are two key files tracking the contents of environments: -``spack.yaml`` and ``spack.lock``. The ``spack.yaml`` file holds -the environment configuration that we previously edited through -``spack config edit``. The ``spack.lock`` file is automatically -generated during concretization. +There are two key files tracking the contents of environments: ``spack.yaml`` and ``spack.lock``. +The ``spack.yaml`` file holds the environment configuration that we previously edited through ``spack config edit``. +The ``spack.lock`` file is automatically generated during concretization. The two files represent two fundamental concepts: @@ -561,9 +469,8 @@ The two files represent two fundamental concepts: * ``spack.lock``: all fully *concrete* specs. -These files are intended to be used by developers and administrators -to manage the environments in a reproducible way. We will cover their -reuse later. +These files are intended to be used by developers and administrators to manage the environments in a reproducible way. +We will cover their reuse later. .. note:: @@ -575,55 +482,45 @@ reuse later. Managed versus independent environments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Environments are either Spack-managed or independent. Both types -of environments are defined by their environment files. So far -we have only created managed environments. This section describes -their differences. +Environments are either Spack-managed or independent. +Both types of environments are defined by their environment files. +So far we have only created managed environments. +This section describes their differences. *Managed environments* are created using ``spack env create ``. -They are automatically created in the ``var/spack/environments`` -subdirectory and can be referenced by their names. +They are automatically created in the ``var/spack/environments`` subdirectory and can be referenced by their names. -*Independent environments* can be created in one of two ways. First, -the Spack environment file(s) can be placed in any directory -(other than ``var/spack/environments``). Alternatively, you can -use ``spack env create -d `` to specify the directory -(````) in which the files should reside. Independent -environments are not named. +*Independent environments* can be created in one of two ways. +First, the Spack environment file(s) can be placed in any directory (other than ``var/spack/environments``). +Alternatively, you can use ``spack env create -d `` to specify the directory (````) in which the files should reside. +Independent environments are not named. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Reviewing a managed environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -We created the ``myproject`` environment earlier using ``spack env -create myproject`` so let's mainly focus on its environment files -in this section. +We created the ``myproject`` environment earlier using ``spack env create myproject`` so let's mainly focus on its environment files in this section. -Earlier, when we changed the environment's configuration using ``spack -config edit``, we were actually editing its ``spack.yaml`` file. We -can move to the directory containing the file using ``spack cd``: +Earlier, when we changed the environment's configuration using ``spack config edit``, we were actually editing its ``spack.yaml`` file. +We can move to the directory containing the file using ``spack cd``: .. literalinclude:: outputs/environments/filenames-1.out :language: console -Notice that ``myproject`` is a subdirectory of ``var/spack/environments`` -within the Spack installation making it a *managed* environment. -Consequently, it can be referenced by name. It will also show up -when running ``spack env list``: +Notice that ``myproject`` is a subdirectory of ``var/spack/environments`` within the Spack installation making it a *managed* environment. +Consequently, it can be referenced by name. +It will also show up when running ``spack env list``: .. literalinclude:: outputs/environments/env-list-2.out :language: console which indicates the active environment by highlighting it in green. -We can also see from the listing above that the current environment directory -contains both of the environment files: ``spack.yaml`` and ``spack.lock``. -This is because ``spack.lock`` was generated when we concretized -the environment. +We can also see from the listing above that the current environment directory contains both of the environment files: ``spack.yaml`` and ``spack.lock``. +This is because ``spack.lock`` was generated when we concretized the environment. -If we ``cat`` the ``spack.yaml`` file, we'll see the same specs and view -options previously shown by ``spack config get``: +If we ``cat`` the ``spack.yaml`` file, we'll see the same specs and view options previously shown by ``spack config get``: .. literalinclude:: outputs/environments/cat-config-1.out :language: console @@ -633,48 +530,41 @@ options previously shown by ``spack config get``: Creating an independent environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Environments do not have to be created in or managed by a Spack -instance. Rather, their environment files can be placed in any -directory. This feature can be quite helpful for use cases such -as environment-based software releases and CI/CD. +Environments do not have to be created in or managed by a Spack instance. +Rather, their environment files can be placed in any directory. +This feature can be quite helpful for use cases such as environment-based software releases and CI/CD. -Let's create an *independent* environment from scratch for a simple -project: +Let's create an *independent* environment from scratch for a simple project: .. literalinclude:: outputs/environments/independent-create-1.out :language: console -Notice that the command shows Spack created the environment, updated -the view, and printed the command needed to activate it. As we -can see in the activation command, since the environment is independent, -it must be referenced by its directory path. +Notice that the command shows Spack created the environment, updated the view, and printed the command needed to activate it. +As we can see in the activation command, since the environment is independent, it must be referenced by its directory path. -Let's see what really happened with this command by listing the -directory contents and looking at the configuration file: +Let's see what really happened with this command by listing the directory contents and looking at the configuration file: .. literalinclude:: outputs/environments/independent-create-2.out :language: console Notice that Spack created a ``spack.yaml`` file in the *code* directory. -Also note that the configuration file has an empty spec list (i.e., -``[]``). That list is intended to contain only the *root specs* of -the environment. +Also note that the configuration file has an empty spec list (i.e., ``[]``). +That list is intended to contain only the *root specs* of the environment. -We can confirm that it is not a managed environment by running -``spack env list``: +We can confirm that it is not a managed environment by running ``spack env list``: .. literalinclude:: outputs/environments/env-list-3.out :language: console and noting that the path does not appear in the output. -Now let's add some specs to the environment. Suppose your project -depends on ``trilinos`` and ``openmpi``. Add these -packages to the spec list using your favorite text editor. The -dash syntax for a YAML list is used in our example. Your package -should now contain the following entries: +Now let's add some specs to the environment. +Suppose your project depends on ``trilinos`` and ``openmpi``. +Add these packages to the spec list using your favorite text editor. +The dash syntax for a YAML list is used in our example. +Your package should now contain the following entries: .. code-block:: yaml @@ -697,18 +587,16 @@ Now activate the environment and install the packages: :language: console -Notice that Spack concretized the specs before installing them and -their dependencies. It also updated the environment's view. Since -we already installed all these packages outside of the environment, -their links were simply added to it. +Notice that Spack concretized the specs before installing them and their dependencies. +It also updated the environment's view. +Since we already installed all these packages outside of the environment, their links were simply added to it. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Updating an installed environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Spack supports tweaking an environment even after the initial specs -are installed. You are free to add and remove specs just as you would -outside of the environment using the command line interface as before. +Spack supports tweaking an environment even after the initial specs are installed. +You are free to add and remove specs just as you would outside of the environment using the command line interface as before. For example, let's add ``hdf5`` and look at our file: @@ -716,8 +604,7 @@ For example, let's add ``hdf5`` and look at our file: :language: console -Notice that ``spack add`` added the package to our active environment and -it appears in the configuration file's spec list. +Notice that ``spack add`` added the package to our active environment and it appears in the configuration file's spec list. .. note:: @@ -730,8 +617,7 @@ Now use ``spack remove`` to remove the spec from the configuration: .. literalinclude:: outputs/environments/remove-independent-1.out :language: console -and we see that the spec *was* removed from the spec list of our -environment. +and we see that the spec *was* removed from the spec list of our environment. .. note:: @@ -744,14 +630,11 @@ Reviewing ``spack.lock`` Now let's turn our attention from the abstract to the concrete. -Our focus so far has been on the abstract environment configuration -represented by the ``spack.yaml`` file. Once that file is concretized, -Spack *generates* a corresponding ``spack.lock`` file representing -the full concretized state of the environment. +Our focus so far has been on the abstract environment configuration represented by the ``spack.yaml`` file. +Once that file is concretized, Spack *generates* a corresponding ``spack.lock`` file representing the full concretized state of the environment. -This file is intended to be a machine-readable representation of the -information needed to *reproduce* the build of an environment. As such, -it is written in ``json``, which is less readable than ``yaml``. +This file is intended to be a machine-readable representation of the information needed to *reproduce* the build of an environment. +As such, it is written in ``json``, which is less readable than ``yaml``. Let's look at the top 30 lines of our current environment: @@ -759,76 +642,58 @@ Let's look at the top 30 lines of our current environment: :language: console -While it is still readable, it consists of over 1900 lines of -information representing the actual configurations for each of -the environment's packages. +While it is still readable, it consists of over 1900 lines of information representing the actual configurations for each of the environment's packages. ^^^^^^^^^^^^^^^^^^^^^^^^^^ Reproducing an environment ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Now that we've described the contents of the environment files we can -discuss how they can be used to reproduce environments. You may want -to do this yourself on a different machine, or use an environment -built by someone else. The process is the same in either case. +Now that we've described the contents of the environment files we can discuss how they can be used to reproduce environments. +You may want to do this yourself on a different machine, or use an environment built by someone else. +The process is the same in either case. -You can recreate an environment by passing either of the environment -files to ``spack env create``. The file you choose depends on whether -you want to approximate the build using the abstract specs or an *exact* -build based on the concrete specs. +You can recreate an environment by passing either of the environment files to ``spack env create``. +The file you choose depends on whether you want to approximate the build using the abstract specs or an *exact* build based on the concrete specs. """""""""""""""""""" Using ``spack.yaml`` """""""""""""""""""" -An approximate build is created using the ``spack.yaml`` file. This -approach is relevant when we want to build the same specs on a new -platform, for example. It allows you to reproduce the environment -by preserving the abstract requirements in the file. However, the -software may actually build differently in part because the concretizer -may choose different dependencies. +An approximate build is created using the ``spack.yaml`` file. +This approach is relevant when we want to build the same specs on a new platform, for example. +It allows you to reproduce the environment by preserving the abstract requirements in the file. +However, the software may actually build differently in part because the concretizer may choose different dependencies. -Let's use ``spack env create`` to create an abstract environment from -the file that we'll call ``abstract``: +Let's use ``spack env create`` to create an abstract environment from the file that we'll call ``abstract``: .. literalinclude:: outputs/environments/create-from-file-1.out :language: console -Here we see that Spack created a managed environment with the name -we provided. +Here we see that Spack created a managed environment with the name we provided. -And, since it is a newly created environment, it does not have any -*installed* specs yet as we can see from calling ``spack find`` -**after** activating the environment: +And, since it is a newly created environment, it does not have any *installed* specs yet as we can see from calling ``spack find`` **after** activating the environment: .. literalinclude:: outputs/environments/find-env-abstract-1.out :language: console -Notice that we have the same root specs as were listed in the ``spack.yaml`` -file. +Notice that we have the same root specs as were listed in the ``spack.yaml`` file. """""""""""""""""""" Using ``spack.lock`` """""""""""""""""""" -The ``spack.lock`` file is used for an exact reproduction of the -original build. It can replicate the build because it contains the -information for all the decisions made during concretization. +The ``spack.lock`` file is used for an exact reproduction of the original build. +It can replicate the build because it contains the information for all the decisions made during concretization. -Now let's create a concrete environment, called ``concrete``, from -the file: +Now let's create a concrete environment, called ``concrete``, from the file: .. literalinclude:: outputs/environments/create-from-file-2.out :language: console -Here we see that Spack again created a managed environment with the -provided name. +Here we see that Spack again created a managed environment with the provided name. -Since we created the environment from our ``spack.lock`` file, -not only do we get the same root specs, all of the packages are -installed in the environment as we can see from calling -``spack find`` **after** activating the environment: +Since we created the environment from our ``spack.lock`` file, not only do we get the same root specs, all of the packages are installed in the environment as we can see from calling ``spack find`` **after** activating the environment: .. literalinclude:: outputs/environments/find-env-concrete-1.out :language: console @@ -842,8 +707,8 @@ installed in the environment as we can see from calling More information ------------------- -This tutorial only scratches the surface of environments and what they -can do. For more information, take a look at the Spack resources below. +This tutorial only scratches the surface of environments and what they can do. +For more information, take a look at the Spack resources below. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Setting up and building environments diff --git a/tutorial_modules.rst b/tutorial_modules.rst index a7420443ea..22caeab09d 100644 --- a/tutorial_modules.rst +++ b/tutorial_modules.rst @@ -11,10 +11,8 @@ Module Files Tutorial ===================== -This tutorial illustrates how Spack can be used to generate module files -for the software that has been installed. Both hierarchical and non-hierarchical -deployments will be discussed in details and we will show how to customize -the content and naming of each module file. +This tutorial illustrates how Spack can be used to generate module files for the software that has been installed. +Both hierarchical and non-hierarchical deployments will be discussed in details and we will show how to customize the content and naming of each module file. At the end of the tutorial readers should have a clear understanding of: @@ -23,8 +21,7 @@ At the end of the tutorial readers should have a clear understanding of: * Which Spack commands can be used to manage module files * How module files generated by Spack can be customized -and be confident that Spack can deal with all of the common use cases -that occur when maintaining software installations on HPC clusters. +and be confident that Spack can deal with all of the common use cases that occur when maintaining software installations on HPC clusters. .. _module_file_tutorial_prerequisites: @@ -32,11 +29,8 @@ that occur when maintaining software installations on HPC clusters. Setup for the Tutorial ---------------------- -To prepare for this tutorial we are going to install a small but representative set -of software that includes different configurations of the same packages and some `external packages -`_. -To keep the installations manageable, let's start by uninstalling everything from -earlier in the tutorial: +To prepare for this tutorial we are going to install a small but representative set of software that includes different configurations of the same packages and some `external packages `_. +To keep the installations manageable, let's start by uninstalling everything from earlier in the tutorial: .. code-block:: console @@ -53,23 +47,21 @@ and by enabling ``tcl`` module files, which are disabled by default since Spack Build a module tool ^^^^^^^^^^^^^^^^^^^ -The first thing that we need is the module tool itself. In the tutorial we will use -``lmod`` because it can work with both hierarchical and non-hierarchical layouts. +The first thing that we need is the module tool itself. +In the tutorial we will use ``lmod`` because it can work with both hierarchical and non-hierarchical layouts. .. code-block:: console $ spack install lmod -Once the module tool is installed we need to have it available in the -current shell. Installation directories in Spack's store are definitely not easy -to remember, but they can be retrieved with the ``spack location`` command: +Once the module tool is installed we need to have it available in the current shell. +Installation directories in Spack's store are definitely not easy to remember, but they can be retrieved with the ``spack location`` command: .. code-block:: console $ . $(spack location -i lmod)/lmod/lmod/init/bash -Now we can re-source the setup file and Spack modules will be put in -our module path. +Now we can re-source the setup file and Spack modules will be put in our module path. .. code-block:: console @@ -78,18 +70,16 @@ our module path. .. FIXME: this needs bootstrap support for ``lmod`` .. FIXME: check the docs here, update them if necessary - If you need to install Lmod or Environment module you can refer - to the documentation `here `_. +If you need to install Lmod or Environment module you can refer to the documentation `here `_. ^^^^^^^^^^^^^^^^^^ Add a new compiler ^^^^^^^^^^^^^^^^^^ -The second step is to build a recent compiler. On first use, Spack -scans the environment and automatically locates the compiler(s) -already available on the system. For this tutorial, however, we want -to use ``gcc@12.3.0``. +The second step is to build a recent compiler. +On first use, Spack scans the environment and automatically locates the compiler(s) already available on the system. +For this tutorial, however, we want to use ``gcc@12.3.0``. .. code-block:: console @@ -102,8 +92,8 @@ You can get this in your environment using ``spack load gcc@12.3.0``: .. literalinclude:: outputs/modules/spack-load-gcc.out :language: console -Now, ``gcc`` is in your ``PATH``. You can add it to the list of -compilers with ``spack compiler add``: +Now, ``gcc`` is in your ``PATH``. +You can add it to the list of compilers with ``spack compiler add``: .. literalinclude:: outputs/modules/add-compiler.out :language: console @@ -113,8 +103,7 @@ To check which compilers are available you can use ``spack compiler list``: .. literalinclude:: outputs/modules/list-compiler.out :language: console -Finally, when you confirmed ``gcc@12.3.0`` is properly registered, clean the environment -with ``spack unload``: +Finally, when you confirmed ``gcc@12.3.0`` is properly registered, clean the environment with ``spack unload``: .. code-block:: console @@ -141,11 +130,10 @@ Finally, we will use Spack to install the packages used in the examples: What are Module Files? ---------------------- -Module files are an easy way to modify your environment in a controlled -manner during a shell session. In general, they contain the information -needed to run an application or use a library. The ``module`` command is -used to interpret and execute module files. For example, ``module show`` -tells you what a module will do when loaded: +Module files are an easy way to modify your environment in a controlled manner during a shell session. +In general, they contain the information needed to run an application or use a library. +The ``module`` command is used to interpret and execute module files. +For example, ``module show`` tells you what a module will do when loaded: .. literalinclude:: outputs/modules/what-are-modules-1.out :language: console @@ -168,16 +156,15 @@ Module Systems ^^^^^^^^^^^^^^ There are two main module systems used in HPC, both installable by Spack. -In this tutorial we will be working with ``lmod`` and be showing examples -with both Tcl and Lua. +In this tutorial we will be working with ``lmod`` and be showing examples with both Tcl and Lua. """"""""""""""""""" Environment Modules """"""""""""""""""" -This is the original modules tool. It can be installed with Spack using the -following command: +This is the original modules tool. +It can be installed with Spack using the following command: .. code-block:: console @@ -185,47 +172,38 @@ following command: It was first coded in C in the early 1990s and was later rewritten entirely in Tcl. -Long stagnant, the project has been revived in the past few years by Xavier -Delaruelle at CEA, and it is now very actively developed. For further details -we refer to its `documentation `_. +Long stagnant, the project has been revived in the past few years by Xavier Delaruelle at CEA, and it is now very actively developed. +For further details we refer to its `documentation `_. """" Lmod """" -Lmod is a module system written in Lua, originally created at the -"Texas Advanced Computing Center" (TACC) by Robert McLay. You can get it with: +Lmod is a module system written in Lua, originally created at the "Texas Advanced Computing Center" (TACC) by Robert McLay. +You can get it with: .. code-block:: console $ spack install lmod as shown in the :ref:`module_file_tutorial_prerequisites` section. -It is a drop-in replacement for Environment Modules, and it works -with *both* Tcl and Lua module files. -It is fully compatible with Environment Modules, but it also -has many distinguishing features of its own. The main one is the -`module hierarchy `_, -which simplifies the module UI by only showing modules built with the -currently loaded compiler and/or MPI. There are also some unique -`safety features `_. +It is a drop-in replacement for Environment Modules, and it works with *both* Tcl and Lua module files. +It is fully compatible with Environment Modules, but it also has many distinguishing features of its own. +The main one is the `module hierarchy `_, which simplifies the module UI by only showing modules built with the currently loaded compiler and/or MPI. +There are also some unique `safety features `_. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ How does Spack generate module files? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Before we dive into the hands-on sections it's worth explaining how -module files are generated by Spack. The following diagram provides a -high-level view of the process: +Before we dive into the hands-on sections it's worth explaining how module files are generated by Spack. +The following diagram provides a high-level view of the process: .. image:: module_file_generation.* -Modules in Spack are generated using configuration files (``config.yaml`` -and ``modules.yaml``), information from Spack's package recipes, and -Jinja2 templates. Spack comes with `Jinja2 -`_, an external template engine, so you -do not need to install it yourself. +Modules in Spack are generated using configuration files (``config.yaml`` and ``modules.yaml``), information from Spack's package recipes, and Jinja2 templates. +Spack comes with `Jinja2 `_, an external template engine, so you do not need to install it yourself. ^^^^^^^^^^^^^^^^^^^^^^^^^ Modules vs ``spack load`` @@ -233,28 +211,24 @@ Modules vs ``spack load`` You may have noticed that we used ``spack load`` in the :ref:`module_file_tutorial_prerequisites` section above. This is a -built-in mechanism of Spack's -- it's designed so that users on a cluster -or a laptop can quickly get a package into their path, and it understands -Spack's spec syntax. It does *not* require modules, as Spack needs to -work regardless of whether modules are set up on the system. +built-in mechanism of Spack's -- it's designed so that users on a cluster or a laptop can quickly get a package into their path, and it understands Spack's spec syntax. +It does *not* require modules, as Spack needs to work regardless of whether modules are set up on the system. -As you might expect, you can see what is loaded via ``spack load`` using -``spack find``: +As you might expect, you can see what is loaded via ``spack load`` using ``spack find``: .. literalinclude:: outputs/modules/show-loaded.out :language: console -Because Spack is designed to be run on HPC systems, it also generates a -module file for every installed package. This allows users unfamiliar -with Spack's interface to see things through the module system they're -used to. To see this, try: +Because Spack is designed to be run on HPC systems, it also generates a module file for every installed package. +This allows users unfamiliar with Spack's interface to see things through the module system they're used to. +To see this, try: .. literalinclude:: outputs/modules/module-avail-1.out :language: console -You can ``module load`` any of these. By default, Spack generates modules -named by ``package-version-compiler-version-hash``, which is a bit hard -to read. We'll show you how to customize this in the following sections. +You can ``module load`` any of these. +By default, Spack generates modules named by ``package-version-compiler-version-hash``, which is a bit hard to read. +We'll show you how to customize this in the following sections. .. _module_file_tutorial_non_hierarchical: @@ -263,32 +237,25 @@ to read. We'll show you how to customize this in the following sections. Non-hierarchical Module Files ----------------------------- -If you arrived to this point you should have an environment that looks -similar to: +If you arrived to this point you should have an environment that looks similar to: .. literalinclude:: outputs/modules/module-avail-2.out :language: console -The non-hierarchical module files that have been generated so far follow -Spack's `default rules for module generation -`_. +The non-hierarchical module files that have been generated so far follow Spack's `default rules for module generation `_. Taking a look at the ``gcc`` module you'll see, for example: .. literalinclude:: outputs/modules/module-show-1.out :language: console -As expected, a few environment variables representing paths will be modified -by the module file according to the default prefix inspection rules. +As expected, a few environment variables representing paths will be modified by the module file according to the default prefix inspection rules. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Filter unwanted modifications to the environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Now consider the case that your site has decided that ``CC``, -``CXX``, ``FC`` and ``F77`` modifications should not be -present in module files. What you can do to abide by the rules is to -create a configuration file ``${SPACK_ROOT}/etc/spack/modules.yaml`` with -the following content: +Now consider the case that your site has decided that ``CC``, ``CXX``, ``FC`` and ``F77`` modifications should not be present in module files. +What you can do to abide by the rules is to create a configuration file ``${SPACK_ROOT}/etc/spack/modules.yaml`` with the following content: .. code-block:: yaml @@ -314,8 +281,7 @@ Next you should regenerate all the module files: .. literalinclude:: outputs/modules/tcl-refresh-1.out :language: console -If you take a look now at the module for ``gcc`` you'll see that the unwanted -paths have disappeared: +If you take a look now at the module for ``gcc`` you'll see that the unwanted paths have disappeared: .. literalinclude:: outputs/modules/module-show-2.out :language: console @@ -324,10 +290,8 @@ paths have disappeared: Prevent some module files from being generated ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Another common request at many sites is to avoid exposing software that -is only needed as an intermediate step when building a newer stack. -Let's try to prevent the generation of -module files for anything that is compiled with ``gcc@11`` (the OS provided compiler). +Another common request at many sites is to avoid exposing software that is only needed as an intermediate step when building a newer stack. +Let's try to prevent the generation of module files for anything that is compiled with ``gcc@11`` (the OS provided compiler). To do this you should add the ``exclude`` keyword to ``${SPACK_ROOT}/etc/spack/modules.yaml``: @@ -348,10 +312,8 @@ To do this you should add the ``exclude`` keyword to ``${SPACK_ROOT}/etc/spack/m - "F77" -and regenerate the module files. This time we'll pass the option -``--delete-tree`` so that Spack will delete the existing module tree and -regenerate a new one, instead of overwriting the files in the existing -directory. +and regenerate the module files. +This time we'll pass the option ``--delete-tree`` so that Spack will delete the existing module tree and regenerate a new one, instead of overwriting the files in the existing directory. .. literalinclude:: outputs/modules/tcl-refresh-2.out :language: console @@ -360,10 +322,8 @@ directory. :language: console -if you look closely you'll see though that we went too far in -excluding modules: the module for ``gcc@12.3.0`` disappeared as it was -bootstrapped with ``gcc@11``. To specify exceptions to the ``exclude`` -rules you can use ``include``: +if you look closely you'll see though that we went too far in excluding modules: the module for ``gcc@12.3.0`` disappeared as it was bootstrapped with ``gcc@11``. +To specify exceptions to the ``exclude`` rules you can use ``include``: .. code-block:: yaml :emphasize-lines: 4,5 @@ -393,9 +353,8 @@ you'll see that now the module for ``gcc@12.3.0`` has reappeared: .. literalinclude:: outputs/modules/module-avail-4.out :language: console -An additional feature that you can leverage to unclutter the environment -is to skip the generation of module files for implicitly installed -packages. In this case you only need to add the following line: +An additional feature that you can leverage to unclutter the environment is to skip the generation of module files for implicitly installed packages. +In this case you only need to add the following line: .. code-block:: yaml :emphasize-lines: 4 @@ -422,10 +381,8 @@ to ``modules.yaml`` and regenerate the module file tree as above. Change module file naming ^^^^^^^^^^^^^^^^^^^^^^^^^ -The next step in making module files more user-friendly is to -improve their naming scheme. -To reduce the length of the hash or remove it altogether you can -use the ``hash_length`` keyword in the configuration file: +The next step in making module files more user-friendly is to improve their naming scheme. +To reduce the length of the hash or remove it altogether you can use the ``hash_length`` keyword in the configuration file: .. code-block:: yaml :emphasize-lines: 4 @@ -458,9 +415,8 @@ If you try to regenerate the module files now you will get an error: about your module files: as a name clash was detected nothing has been changed on disk. -The problem here is that without the hashes the four different flavors of -``netlib-scalapack`` map to the same module file name. We can change how -the names are formatted to differentiate them: +The problem here is that without the hashes the four different flavors of ``netlib-scalapack`` map to the same module file name. +We can change how the names are formatted to differentiate them: .. code-block:: yaml :emphasize-lines: 10-11,18-21 @@ -487,16 +443,10 @@ the names are formatted to differentiate them: netlib-scalapack: '{name}/{version}-{compiler.name}-{compiler.version}-{^lapack.name}-{^mpi.name}' ^python^lapack: '{name}/{version}-{compiler.name}-{compiler.version}-{^lapack.name}' -As you can see it is possible to specify rules that apply only to a -restricted set of packages using `anonymous specs -`_ -like ``^python^lapack``. Here we declare a conflict between any two modules -with the same name, so they cannot be loaded together. We also format the -names of modules according to compiler, compiler version, and MPI provider -name using the `spec format syntax -`_. -This allows us to match specs by their dependencies, and format them -based on their DAGs. +As you can see it is possible to specify rules that apply only to a restricted set of packages using `anonymous specs `_ like ``^python^lapack``. +Here we declare a conflict between any two modules with the same name, so they cannot be loaded together. +We also format the names of modules according to compiler, compiler version, and MPI provider name using the `spec format syntax `_. +This allows us to match specs by their dependencies, and format them based on their DAGs. .. literalinclude:: outputs/modules/tcl-refresh-5.out :language: console @@ -512,10 +462,8 @@ based on their DAGs. Add custom environment modifications ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -At many sites it is customary to set an environment variable in a -package's module file that points to the folder in which the package -is installed. You can achieve this with Spack by adding an -``environment`` directive to the configuration file: +At many sites it is customary to set an environment variable in a package's module file that points to the folder in which the package is installed. +You can achieve this with Spack by adding an ``environment`` directive to the configuration file: .. code-block:: yaml :emphasize-lines: 19-21 @@ -547,8 +495,8 @@ is installed. You can achieve this with Spack by adding an ^python^lapack: '{name}/{version}-{compiler.name}-{compiler.version}-{^lapack.name}' -Under the hood Spack uses the :meth:`~spack.spec.Spec.format` API to substitute -tokens in either environment variable names or values. There are two caveats though: +Under the hood Spack uses the :meth:`~spack.spec.Spec.format` API to substitute tokens in either environment variable names or values. +There are two caveats though: - The set of allowed tokens in variable names is restricted to ``name``, ``version``, ``compiler``, ``compiler.name``, @@ -566,9 +514,8 @@ Regenerating the module files results in something like: As you can see, the ``gcc`` module has the environment variable ``GCC_ROOT`` set. -Sometimes it's also useful to apply environment modifications selectively and target -only certain packages. You can for instance apply modifications to the -``openmpi`` module as follows: +Sometimes it's also useful to apply environment modifications selectively and target only certain packages. +You can for instance apply modifications to the ``openmpi`` module as follows: .. code-block:: yaml :emphasize-lines: 22-26 @@ -619,10 +566,8 @@ This time we will be more selective and regenerate only the ``openmpi`` module f Autoload dependencies ^^^^^^^^^^^^^^^^^^^^^ -Spack can also generate module files that contain code to load the -dependencies automatically. You can, for instance generate python -modules that load their dependencies by adding the ``autoload`` -directive and assigning it the value ``direct``: +Spack can also generate module files that contain code to load the dependencies automatically. +You can, for instance generate python modules that load their dependencies by adding the ``autoload`` directive and assigning it the value ``direct``: .. code-block:: yaml :emphasize-lines: 4,32,33 @@ -671,23 +616,19 @@ and will contain code to autoload all the dependencies: .. literalinclude:: outputs/modules/load-direct.out :language: console -In case messages are unwanted during the autoload procedure, it will be -sufficient to omit the line setting ``verbose: true`` in the configuration file above. +In case messages are unwanted during the autoload procedure, it will be sufficient to omit the line setting ``verbose: true`` in the configuration file above. ------------------------- Hierarchical Module Files ------------------------- -So far we worked with non-hierarchical module files, i.e. with module files -that are all generated in the same root directory and don't attempt to -dynamically modify the ``MODULEPATH``. This results in a flat module structure where -all the software is visible at the same time: +So far we worked with non-hierarchical module files, i.e. with module files that are all generated in the same root directory and don't attempt to dynamically modify the ``MODULEPATH``. +This results in a flat module structure where all the software is visible at the same time: .. literalinclude:: outputs/modules/lmod-intro-avail.out :language: console -This layout is quite simple to deploy, but you can see from the above snippet -that nothing prevents users from loading incompatible sets of modules: +This layout is quite simple to deploy, but you can see from the above snippet that nothing prevents users from loading incompatible sets of modules: .. literalinclude:: outputs/modules/lmod-intro-conflict.out :language: console @@ -697,20 +638,15 @@ Even if ``conflicts`` directives are carefully placed in module files, they: - won't enforce a consistent environment, but will just report an error - need constant updates, for instance as soon as a new compiler or MPI library is installed -`Hierarchical module files `_ try to -overcome these shortcomings by showing at start-up only a restricted view of what is -available on the system: more specifically only the software that has been installed with -OS provided compilers. Among this software there will be other - usually more recent - compilers -that, once loaded, will prepend new directories to ``MODULEPATH`` unlocking all the software -that was compiled with them. This "unlocking" idea can then be extended arbitrarily to -virtual dependencies, as we'll see in the following section. +`Hierarchical module files `_ try to overcome these shortcomings by showing at start-up only a restricted view of what is available on the system: more specifically only the software that has been installed with OS provided compilers. +Among this software there will be other - usually more recent - compilers that, once loaded, will prepend new directories to ``MODULEPATH`` unlocking all the software that was compiled with them. +This "unlocking" idea can then be extended arbitrarily to virtual dependencies, as we'll see in the following section. ^^^^^^^^^^^^^^^^^ Core/Compiler/MPI ^^^^^^^^^^^^^^^^^ -The most widely used hierarchy is the so called ``Core/Compiler/MPI`` where, on top -of the compilers, different MPI libraries also unlock software linked to them. +The most widely used hierarchy is the so called ``Core/Compiler/MPI`` where, on top of the compilers, different MPI libraries also unlock software linked to them. There are just a few steps needed to adapt the ``modules.yaml`` file we used previously: #. enable the ``lmod`` file generator @@ -764,11 +700,9 @@ After these modifications your configuration file should look like: that only ``lmod`` will be active (see `Overriding entire sections `_ for more details). -The directive ``core_compilers`` accepts a list of compilers. Everything built -using these compilers will create a module in the ``Core`` part of the hierarchy, -which is the entry point for hierarchical module files. It is -common practice to put the OS provided compilers in the list and only build common utilities -and other compilers with them. +The directive ``core_compilers`` accepts a list of compilers. +Everything built using these compilers will create a module in the ``Core`` part of the hierarchy, which is the entry point for hierarchical module files. +It is common practice to put the OS provided compilers in the list and only build common utilities and other compilers with them. If we now regenerate the module files: @@ -788,14 +722,14 @@ asking for the available modules will return: .. literalinclude:: outputs/modules/module-avail-6.out :language: console -Unsurprisingly, the only visible module is ``gcc``. Loading that we'll unlock -the ``Compiler`` part of the hierarchy: +Unsurprisingly, the only visible module is ``gcc``. +Loading that we'll unlock the ``Compiler`` part of the hierarchy: .. literalinclude:: outputs/modules/module-avail-7.out :language: console -The same holds true also for the ``MPI`` part, that you can enable by loading -either ``mpich`` or ``openmpi``. Let's start by loading ``mpich``: +The same holds true also for the ``MPI`` part, that you can enable by loading either ``mpich`` or ``openmpi``. +Let's start by loading ``mpich``: .. literalinclude:: outputs/modules/module-avail-8.out :language: console @@ -803,16 +737,13 @@ either ``mpich`` or ``openmpi``. Let's start by loading ``mpich``: .. literalinclude:: outputs/modules/module-load-openblas-scalapack.out :language: console -At this point we can showcase the improved consistency that a hierarchical layout provides -over a non-hierarchical one: +At this point we can showcase the improved consistency that a hierarchical layout provides over a non-hierarchical one: .. literalinclude:: outputs/modules/module-swap-mpi.out :language: console -``Lmod`` took care of swapping the MPI provider for us, and it also substituted the -``netlib-scalapack`` module to conform to the change in the MPI. -In this way we can't accidentally pull-in two different MPI providers at the -same time or load a module file for a package linked to ``openmpi`` when ``mpich`` is also loaded. +``Lmod`` took care of swapping the MPI provider for us, and it also substituted the ``netlib-scalapack`` module to conform to the change in the MPI. +In this way we can't accidentally pull-in two different MPI providers at the same time or load a module file for a package linked to ``openmpi`` when ``mpich`` is also loaded. Consistency for compilers and MPI is ensured by the tool. @@ -820,40 +751,29 @@ Consistency for compilers and MPI is ensured by the tool. Add LAPACK to the hierarchy ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The hierarchy just shown is already a great improvement over non-hierarchical layouts, -but it still has an asymmetry: ``LAPACK`` providers cover the same semantic role -as ``MPI`` providers, but yet they are not part of the hierarchy. +The hierarchy just shown is already a great improvement over non-hierarchical layouts, but it still has an asymmetry: ``LAPACK`` providers cover the same semantic role as ``MPI`` providers, but yet they are not part of the hierarchy. -To be more practical, this means that although we have gained an improved consistency in -our environment when it comes to ``MPI``, we still have the same problems as we had before -for ``LAPACK`` implementations: +To be more practical, this means that although we have gained an improved consistency in our environment when it comes to ``MPI``, we still have the same problems as we had before for ``LAPACK`` implementations: .. literalinclude:: outputs/modules/lapack-conflict.out :language: console -Hierarchies that are deeper than ``Core``/``Compiler``/``MPI`` are -probably still considered "unusual" or "impractical" at many sites, mainly because -module files are written manually and keeping track of the combinations -among multiple providers quickly becomes quite involved. +Hierarchies that are deeper than ``Core``/``Compiler``/``MPI`` are probably still considered "unusual" or "impractical" at many sites, mainly because module files are written manually and keeping track of the combinations among multiple providers quickly becomes quite involved. -For instance, having both ``MPI`` and ``LAPACK`` in the hierarchy -means we must classify software into one of four categories: +For instance, having both ``MPI`` and ``LAPACK`` in the hierarchy means we must classify software into one of four categories: #. Software that doesn't depend on ``MPI`` or ``LAPACK`` #. Software that depends only on ``MPI`` #. Software that depends only on ``LAPACK`` #. Software that depends on both -to decide when to show it to the user. The situation becomes more involved as the number of virtual -dependencies in the hierarchy increases. +to decide when to show it to the user. +The situation becomes more involved as the number of virtual dependencies in the hierarchy increases. -We can take advantage of the DAG that Spack maintains for the installed software and solve -this combinatorial problem in a clean and automated way. -In some sense Spack's ability to manage this combinatorial complexity makes deeper -hierarchies feasible. +We can take advantage of the DAG that Spack maintains for the installed software and solve this combinatorial problem in a clean and automated way. +In some sense Spack's ability to manage this combinatorial complexity makes deeper hierarchies feasible. -Coming back to our example, let's add ``lapack`` to the hierarchy and -remove the remaining suffix projection for ``lapack``: +Coming back to our example, let's add ``lapack`` to the hierarchy and remove the remaining suffix projection for ``lapack``: .. code-block:: yaml :emphasize-lines: 10 @@ -905,49 +825,36 @@ Both ``MPI`` and ``LAPACK`` providers will now benefit from the same safety feat .. literalinclude:: outputs/modules/lapack-correct.out :language: console -Because we only compiled ``py-numpy`` with ``openblas`` the module -is made inactive when we switch the ``LAPACK`` provider. The user -environment is now consistent by design! +Because we only compiled ``py-numpy`` with ``openblas`` the module is made inactive when we switch the ``LAPACK`` provider. +The user environment is now consistent by design! ---------------------- Working with Templates ---------------------- -As briefly mentioned in the introduction, Spack uses `Jinja2 `_ -to generate each individual module file. -This means that you have all of its flexibility and power when it comes to -customizing what gets generated! +As briefly mentioned in the introduction, Spack uses `Jinja2 `_ to generate each individual module file. +This means that you have all of its flexibility and power when it comes to customizing what gets generated! ^^^^^^^^^^^^^^^^^^^^^ Module file templates ^^^^^^^^^^^^^^^^^^^^^ -The templates that Spack uses to generate module files are stored in the -``share/spack/templates/module`` directory within the Spack prefix, and -they all share the same common structure. Usually, they start with a -header that identifies the type of module being generated. In the case of -hierarchical module files it's: +The templates that Spack uses to generate module files are stored in the ``share/spack/templates/module`` directory within the Spack prefix, and they all share the same common structure. +Usually, they start with a header that identifies the type of module being generated. +In the case of hierarchical module files it's: .. literalinclude:: _spack_root/share/spack/templates/modules/modulefile.lua :language: jinja :lines: 1-6 -The statements within double curly brackets ``{{ ... }}`` denote -`expressions `_ -that will be evaluated and substituted at module generation time. -The rest of the file is then divided into -`blocks `_ -that can be overridden or extended by users, if need be. -`Control structures `_ -, delimited by ``{% ... %}``, -are also permitted in the template language: +The statements within double curly brackets ``{{ ... }}`` denote `expressions `_ that will be evaluated and substituted at module generation time. +The rest of the file is then divided into `blocks `_ that can be overridden or extended by users, if need be. `Control structures `_ , delimited by ``{% ... %}``, are also permitted in the template language: .. literalinclude:: _spack_root/share/spack/templates/modules/modulefile.lua :language: jinja :lines: 73-87 -The locations where Spack looks for templates are specified -in ``config.yaml``: +The locations where Spack looks for templates are specified in ``config.yaml``: .. literalinclude:: _spack_root/etc/spack/defaults/config.yaml :language: yaml @@ -959,15 +866,11 @@ and can be extended by users to employ custom templates, as we'll see next. Extend the default templates ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Let's assume one of our software is protected by group membership: -allowed users belong to the same linux group, and access is granted at group level. -Wouldn't it be nice if people that are not -yet entitled to use it could receive a helpful message at module load time -that tells them who to contact in your organization to be inserted in the group? +Let's assume one of our software is protected by group membership: allowed users belong to the same linux group, and access is granted at group level. +Wouldn't it be nice if people that are not yet entitled to use it could receive a helpful message at module load time that tells them who to contact in your organization to be inserted in the group? -To automate the generation of module files with such site-specific behavior -we'll start by extending the list of locations where Spack looks for module -files. Let's create the file ``${SPACK_ROOT}/etc/spack/config.yaml`` with the content: +To automate the generation of module files with such site-specific behavior we'll start by extending the list of locations where Spack looks for module files. +Let's create the file ``${SPACK_ROOT}/etc/spack/config.yaml`` with the content: .. code-block:: yaml @@ -1013,9 +916,8 @@ The section: {% endblock %} overrides the ``footer`` block. -Finally, we need to add a couple of lines in ``modules.yaml`` to tell Spack which specs -need to use the new custom template. For the sake of illustration let's assume -it's ``netlib-scalapack``: +Finally, we need to add a couple of lines in ``modules.yaml`` to tell Spack which specs need to use the new custom template. +For the sake of illustration let's assume it's ``netlib-scalapack``: .. code-block:: yaml :emphasize-lines: 30-31 @@ -1071,17 +973,14 @@ we'll find the following at the end of each ``netlib-scalapack`` module file: ) end -and every user that doesn't have access to the software will now be redirected to -the right e-mail address where to ask for it! +and every user that doesn't have access to the software will now be redirected to the right e-mail address where to ask for it! ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Restore settings for future sections ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -For future sections of the tutorial, we will not use the ``gcc@12.3.0`` -compiler. Since it is currently the default compiler (our current -default is the most recent version of gcc available), we will remove -it now. +For future sections of the tutorial, we will not use the ``gcc@12.3.0`` compiler. +Since it is currently the default compiler (our current default is the most recent version of gcc available), we will remove it now. .. code-block:: console diff --git a/tutorial_packaging.rst b/tutorial_packaging.rst index 5e5aef51bf..00746b7f06 100644 --- a/tutorial_packaging.rst +++ b/tutorial_packaging.rst @@ -11,19 +11,15 @@ Package Creation Tutorial ========================= -This tutorial walks you through the steps for creating and -debugging a simple Spack package. We will develop and debug -a package using an iterative approach in order to gain more -experience with additional Spack commands. For consistency, -we will create the package for ``mpileaks`` (https://github.com/LLNL/mpileaks), -which is an MPI debugging tool. +This tutorial walks you through the steps for creating and debugging a simple Spack package. +We will develop and debug a package using an iterative approach in order to gain more experience with additional Spack commands. +For consistency, we will create the package for ``mpileaks`` (https://github.com/LLNL/mpileaks), which is an MPI debugging tool. ------------------------ What is a Spack Package? ------------------------ -Spack packages are installation scripts, which are essentially -recipes for building (and testing) software. +Spack packages are installation scripts, which are essentially recipes for building (and testing) software. They define properties and behavior of the build, such as: @@ -32,27 +28,21 @@ They define properties and behavior of the build, such as: * options for building from source; and * build commands. -They can also define checks of the installed software that can -be performed after the installation. +They can also define checks of the installed software that can be performed after the installation. -Once we've specified a package's recipe, users can ask Spack to -build the software with different features on any of the supported -systems. +Once we've specified a package's recipe, users can ask Spack to build the software with different features on any of the supported systems. --------------- Getting Started --------------- -In order to avoid modifying your Spack installation with the package we -are creating, add a **package repository** just for this tutorial by -entering the following command: +In order to avoid modifying your Spack installation with the package we are creating, add a **package repository** just for this tutorial by entering the following command: .. literalinclude:: outputs/packaging/repo-add.out :language: console -Doing this ensures changes we make here do not adversely affect other -parts of the tutorial. You can find out more about repositories at -`Package Repositories `_. +Doing this ensures changes we make here do not adversely affect other parts of the tutorial. +You can find out more about repositories at `Package Repositories `_. ------------------------- Creating the Package File @@ -64,12 +54,10 @@ Creating the Package File is set to the name or path of your preferred text editor. -Suppose you want to install software that depends on mpileaks but found -Spack did not already have a built-in package for it. This means you are -going to have to create one. +Suppose you want to install software that depends on mpileaks but found Spack did not already have a built-in package for it. +This means you are going to have to create one. -Spack's *create* command builds a new package from a template by taking -the location of the package's source code and using it to: +Spack's *create* command builds a new package from a template by taking the location of the package's source code and using it to: * fetch the code; * create a package skeleton; and @@ -81,25 +69,19 @@ the location of the package's source code and using it to: versions can be found at `Creating and Editing Packages `_. -The ``mpileaks`` source code is available in a tarball in the -software's repository (https://github.com/LLNL/mpileaks). Spack -will look at the contents of the tarball and generate a package when we -run ``spack create`` with the URL: +The ``mpileaks`` source code is available in a tarball in the software's repository (https://github.com/LLNL/mpileaks). +Spack will look at the contents of the tarball and generate a package when we run ``spack create`` with the URL: .. literalinclude:: outputs/packaging/create.out :language: console -You should now be in your text editor of choice, with the ``package.py`` -file open for editing. +You should now be in your text editor of choice, with the ``package.py`` file open for editing. -Your ``package.py`` file should reside in the ``tutorial-mpileaks`` -subdirectory of your tutorial repository's ``packages`` directory, i.e., -``$SPACK_ROOT/var/spack/repos/tutorial/packages/tutorial-mpileaks/package.py`` +Your ``package.py`` file should reside in the ``tutorial-mpileaks`` subdirectory of your tutorial repository's ``packages`` directory, i.e., ``$SPACK_ROOT/var/spack/repos/tutorial/packages/tutorial-mpileaks/package.py`` Take a moment to look over the file. -As we can see from the skeleton contents, shown below, the Spack -template: +As we can see from the skeleton contents, shown below, the Spack template: * provides instructions for how to contribute your package to the Spack repository; @@ -124,9 +106,8 @@ template: maintain a Spack package for their own software and/or rely on software maintained by other people. -Since we are providing a ``url``, we can confirm the checksum, or ``sha256`` -calculation. Exit your editor to return to the command line and use the -``spack checksum`` command: +Since we are providing a ``url``, we can confirm the checksum, or ``sha256`` calculation. +Exit your editor to return to the command line and use the ``spack checksum`` command: .. literalinclude:: outputs/packaging/checksum-mpileaks-1.out :language: console @@ -139,14 +120,13 @@ We will now fill in the provided placeholders as we: * add dependencies; and * add the configuration arguments needed to build the package. -For the moment, though, let's see what Spack does with the skeleton -by trying to install the package using the ``spack install`` command: +For the moment, though, let's see what Spack does with the skeleton by trying to install the package using the ``spack install`` command: .. literalinclude:: outputs/packaging/install-mpileaks-1.out :language: console -It clearly did not build. The error indicates ``configure`` is unable -to find the installation location of a dependency. +It clearly did not build. +The error indicates ``configure`` is unable to find the installation location of a dependency. So let's start to customize the package for our software. @@ -156,8 +136,7 @@ Adding Package Documentation First, let's fill in the documentation. -Bring mpileaks' ``package.py`` file back into your ``$EDITOR`` with the -``spack edit`` command: +Bring mpileaks' ``package.py`` file back into your ``$EDITOR`` with the ``spack edit`` command: .. code-block:: console @@ -189,16 +168,15 @@ The resulting package should contain the following information: :emphasize-lines: 5,7,10,12 At this point we've only updated key documentation within the package. -It won't help us build the software but the information is now available -for review. +It won't help us build the software but the information is now available for review. Let's enter the ``spack info`` command for the package: .. literalinclude:: outputs/packaging/info-mpileaks.out :language: console -Take a moment to look over the output. You should see the following -information derived from the package: +Take a moment to look over the output. +You should see the following information derived from the package: * it is an Autotools package; * it has the description, homepage, and maintainer(s) we provided; @@ -209,8 +187,7 @@ information derived from the package: * both the link and run dependencies are ``None`` at this point; and * it uses the 3-clause BSD license. -As we fill in more information about the package, the ``spack info`` -command will become more informative. +As we fill in more information about the package, the ``spack info`` command will become more informative. .. note:: @@ -233,9 +210,8 @@ Now we're ready to start filling in the build recipe. Adding Dependencies ------------------- -First we'll add the dependencies determined by reviewing documentation -in the software's repository (https://github.com/LLNL/mpileaks). The -``mpileaks`` software relies on three third-party libraries: +First we'll add the dependencies determined by reviewing documentation in the software's repository (https://github.com/LLNL/mpileaks). +The ``mpileaks`` software relies on three third-party libraries: * ``mpi``, * ``adept-utils``, and @@ -246,15 +222,13 @@ in the software's repository (https://github.com/LLNL/mpileaks). The Luckily, all of these dependencies are built-in packages in Spack; otherwise, we would have to create packages for them as well. -Bring mpileaks' ``package.py`` file back up in your ``$EDITOR`` with -the ``spack edit`` command: +Bring mpileaks' ``package.py`` file back up in your ``$EDITOR`` with the ``spack edit`` command: .. code-block:: console $ spack edit tutorial-mpileaks -and add the dependencies by specifying them using the ``depends_on`` -directive as shown below: +and add the dependencies by specifying them using the ``depends_on`` directive as shown below: .. literalinclude:: tutorial/examples/packaging/2.package.py :caption: tutorial-mpileaks/package.py (from tutorial/examples/packaging/2.package.py) @@ -262,8 +236,7 @@ directive as shown below: :language: python :emphasize-lines: 25-27 -Adding dependencies tells Spack that it must ensure these packages are -installed *before* it can build our package. +Adding dependencies tells Spack that it must ensure these packages are installed *before* it can build our package. .. note:: @@ -298,13 +271,11 @@ It found that: Debugging Package Builds ------------------------ -Our ``tutorial-mpileaks`` package is still not building due to the -``adept-utils`` package's ``configure`` error. Experienced -Autotools developers will likely already see the problem and -its solution. +Our ``tutorial-mpileaks`` package is still not building due to the ``adept-utils`` package's ``configure`` error. +Experienced Autotools developers will likely already see the problem and its solution. -But let's take this opportunity to use Spack features to investigate -the problem. Our options for proceeding are: +But let's take this opportunity to use Spack features to investigate the problem. +Our options for proceeding are: * review the build log; and * build the package manually. @@ -313,19 +284,15 @@ the problem. Our options for proceeding are: Reviewing the Build Log ~~~~~~~~~~~~~~~~~~~~~~~ -The build log might yield some clues so let's look at the contents of -the ``spack-build-out.txt`` file at the path recommended above by our -failed installation: +The build log might yield some clues so let's look at the contents of the ``spack-build-out.txt`` file at the path recommended above by our failed installation: .. literalinclude:: outputs/packaging/build-output.out :language: console -In this case the error conveniently appears on the last line of the -log *and* the output from `spack install`. +In this case the error conveniently appears on the last line of the log *and* the output from `spack install`. Here we also see a number of checks performed by the ``configure`` command. -Most importantly, the last line is very clear: the installation path of the -``adept-utils`` dependency cannot be found. +Most importantly, the last line is very clear: the installation path of the ``adept-utils`` dependency cannot be found. .. note:: @@ -340,8 +307,7 @@ So let's investigate further from the staged build directory. Building Manually ~~~~~~~~~~~~~~~~~ -First let's try to build the package manually to see if we can -figure out how to solve the problem. +First let's try to build the package manually to see if we can figure out how to solve the problem. Let's move to the build directory using the ``spack cd`` command: @@ -349,54 +315,42 @@ Let's move to the build directory using the ``spack cd`` command: $ spack cd tutorial-mpileaks -You should now be in the appropriate stage directory since this -command moves us into the working directory of the last attempted -build. If not, you can ``cd`` into the directory above that contained -the ``spack-build-out.txt`` file then into it's ``spack-src`` -subdirectory. +You should now be in the appropriate stage directory since this command moves us into the working directory of the last attempted build. +If not, you can ``cd`` into the directory above that contained the ``spack-build-out.txt`` file then into it's ``spack-src`` subdirectory. -Now let's ensure the environment is properly set up using the -``spack build-env`` command: +Now let's ensure the environment is properly set up using the ``spack build-env`` command: .. code-block:: console $ spack build-env tutorial-mpileaks bash -This command spawned a new shell containing the same environment -that Spack used to build the ``tutorial-mpileaks`` package. (Feel -free to substitute your favorite shell for ``bash``.) +This command spawned a new shell containing the same environment that Spack used to build the ``tutorial-mpileaks`` package. (Feel free to substitute your favorite shell for ``bash``.) .. note:: If you are running using an AWS instance, you'll want to substitute your home directory for ``/home/spack`` below. -From here we can manually re-run the build using the ``configure`` -command: +From here we can manually re-run the build using the ``configure`` command: .. literalinclude:: outputs/packaging/build-env-configure.out :language: console -And we get the same results as before. Unfortunately, the output -does not provide any additional information that can help us with -the build. +And we get the same results as before. +Unfortunately, the output does not provide any additional information that can help us with the build. -Given that this is a simple package built with ``configure`` and we know -that installation directories need to be specified, we can use its -help to see what command line options are available for the software. +Given that this is a simple package built with ``configure`` and we know that installation directories need to be specified, we can use its help to see what command line options are available for the software. .. literalinclude:: outputs/packaging/configure-help.out :language: console :emphasize-lines: 80-81 -Note that you can specify the paths for the two concrete dependencies -with the following options: +Note that you can specify the paths for the two concrete dependencies with the following options: * ``--with-adept-utils=PATH`` * ``--with-callpath=PATH`` -So let's leave the spawned shell and return to the Spack repository -directory: +So let's leave the spawned shell and return to the Spack repository directory: .. code-block:: console @@ -409,28 +363,22 @@ Now that we know what arguments to provide, we can update the recipe. Specifying Configure Arguments ------------------------------ -We now know which options we need to pass to ``configure``, but how do we -know where to find the installation paths for the package's dependencies -from within the ``package.py`` file? +We now know which options we need to pass to ``configure``, but how do we know where to find the installation paths for the package's dependencies from within the ``package.py`` file? Fortunately, we can query the package's concrete ``Spec`` instance. The ``self.spec`` property holds the package's directed acyclic graph (DAG) of its dependencies. Each dependency's ``Spec``, accessed by name, has a ``prefix`` property containing its installation path. -So let's add the configuration arguments for specifying the paths to -the two concrete dependencies in the ``configure_args`` method of our -package. +So let's add the configuration arguments for specifying the paths to the two concrete dependencies in the ``configure_args`` method of our package. -Bring mpileaks' ``package.py`` file back up in your ``$EDITOR`` with -the ``spack edit`` command: +Bring mpileaks' ``package.py`` file back up in your ``$EDITOR`` with the ``spack edit`` command: .. code-block:: console $ spack edit tutorial-mpileaks -and add the ``--with-adept-utils`` and ``--with-callpath`` arguments -in the ``configure_args`` method as follows: +and add the ``--with-adept-utils`` and ``--with-callpath`` arguments in the ``configure_args`` method as follows: .. literalinclude:: tutorial/examples/packaging/3.package.py :caption: tutorial-mpileaks/package.py (from tutorial/examples/packaging/3.package.py) @@ -438,8 +386,7 @@ in the ``configure_args`` method as follows: :language: python :emphasize-lines: 32-36 -Since this is an ``AutotoolsPackage``, the arguments returned from the -method will automatically get passed to ``configure`` during the build. +Since this is an ``AutotoolsPackage``, the arguments returned from the method will automatically get passed to ``configure`` during the build. Now let's try the build again: @@ -448,8 +395,7 @@ Now let's try the build again: Success! -All we needed to do was add the path arguments for the two concrete -packages for configure to perform a simple, no frills build. +All we needed to do was add the path arguments for the two concrete packages for configure to perform a simple, no frills build. But is that all we can do to help other users build our software? @@ -460,20 +406,14 @@ Adding Variants What if we want to expose the software's optional features in the package? We can do this by adding build-time options using package *variants*. -Recall from configure's help output for ``tutorial-mpileaks`` that the -software has several optional features and packages that we could support -in Spack. Two stand out for tutorial purposes because they both take integers, -as opposed to simply allowing them to be enabled or disabled. +Recall from configure's help output for ``tutorial-mpileaks`` that the software has several optional features and packages that we could support in Spack. +Two stand out for tutorial purposes because they both take integers, as opposed to simply allowing them to be enabled or disabled. .. literalinclude:: outputs/packaging/configure-build-options.out :language: console :emphasize-lines: 18-23 -According to the software's documentation (https://github.com/LLNL/mpileaks), -the integer values for the ``--with-stack-start-*`` options represent the -numbers of calls to shave off of the top of the stack traces for each -language, effectively reducing the noise of internal mpileaks library function -calls in generated traces. +According to the software's documentation (https://github.com/LLNL/mpileaks), the integer values for the ``--with-stack-start-*`` options represent the numbers of calls to shave off of the top of the stack traces for each language, effectively reducing the noise of internal mpileaks library function calls in generated traces. For simplicity, we'll use one variant to supply the value for both arguments. @@ -482,13 +422,11 @@ Supporting this optional feature will require two changes to the package: * add a ``variant`` directive; and * change the configure options to use the value. -Let's add the variant to expect an ``int`` value with a default of -``0``. Defaulting to ``0`` effectively disables the option. Also change -``configure_args`` to retrieve the value and add the corresponding -configure arguments when a non-zero value is provided by the user. +Let's add the variant to expect an ``int`` value with a default of ``0``. +Defaulting to ``0`` effectively disables the option. +Also change ``configure_args`` to retrieve the value and add the corresponding configure arguments when a non-zero value is provided by the user. -Bring mpileaks' ``package.py`` file back up in your ``$EDITOR`` with -the ``spack edit`` command: +Bring mpileaks' ``package.py`` file back up in your ``$EDITOR`` with the ``spack edit`` command: .. code-block:: console @@ -502,35 +440,28 @@ and add the ``variant`` directive and associated arguments as follows: :language: python :emphasize-lines: 16-21,45-52 -Notice that the ``variant`` directive is translated into a ``variants`` dictionary -in ``self.spec``. Also note that the value provided by the user is accessed -by the entry's ``value`` property. +Notice that the ``variant`` directive is translated into a ``variants`` dictionary in ``self.spec``. +Also note that the value provided by the user is accessed by the entry's ``value`` property. -Now run the installation again with the ``--verbose`` install option -- to -get more output during the build -- and the new ``stackstart`` package option: +Now run the installation again with the ``--verbose`` install option -- to get more output during the build -- and the new ``stackstart`` package option: .. literalinclude:: outputs/packaging/install-mpileaks-4.out :language: console -Notice the addition of the two stack start arguments in the configure -command that appears at the end of the highlighted line after mpileaks' -``Executing phase: 'configure'``. +Notice the addition of the two stack start arguments in the configure command that appears at the end of the highlighted line after mpileaks' ``Executing phase: 'configure'``. ------------ Adding Tests ------------ -The simplest tests we can add are sanity checks, which can be used to -ensure the directories and files we expect to be installed for all -versions of the package actually exist. If we look at a successful -installation, we can see that the following directories will be installed: +The simplest tests we can add are sanity checks, which can be used to ensure the directories and files we expect to be installed for all versions of the package actually exist. +If we look at a successful installation, we can see that the following directories will be installed: * bin * lib * share -So let's add a simple sanity check to ensure they are present, BUT let's -enter a typo to see what happens: +So let's add a simple sanity check to ensure they are present, BUT let's enter a typo to see what happens: .. literalinclude:: tutorial/examples/packaging/5.package.py :caption: tutorial-mpileaks/package.py (from tutorial/examples/packaging/5.package.py) @@ -558,23 +489,20 @@ Installing again we can see we've fixed the problem. .. literalinclude:: outputs/packaging/install-mpileaks-6.out :language: console -This is just scratching the surface of testing an installation. We could -leverage the examples from this package to add post-install phase tests -and/or stand-lone tests. Refer to the links at the bottom for more -information on checking an installation. +This is just scratching the surface of testing an installation. +We could leverage the examples from this package to add post-install phase tests and/or stand-lone tests. +Refer to the links at the bottom for more information on checking an installation. ------------------------ Querying the Spec Object ------------------------ -As packages evolve and are ported to different systems, build recipes -often need to change as well. This is where the package's ``Spec`` comes -in. +As packages evolve and are ported to different systems, build recipes often need to change as well. +This is where the package's ``Spec`` comes in. -So far we've looked at getting the paths for dependencies and values of -variants from the ``Spec`` but there is more. The package's ``self.spec``, -property allows you to query information about the package build, such as: +So far we've looked at getting the paths for dependencies and values of variants from the ``Spec`` but there is more. +The package's ``self.spec``, property allows you to query information about the package build, such as: * how a package's dependencies were built; * what compiler was being used; @@ -587,8 +515,8 @@ Examples of common queries are provided below. Querying Spec Versions ~~~~~~~~~~~~~~~~~~~~~~ -You can customize the build based on the version of the package, compiler, -and dependencies. Examples of each are: +You can customize the build based on the version of the package, compiler, and dependencies. +Examples of each are: * Am I building my package with version ``1.1`` or greater? @@ -615,8 +543,8 @@ and dependencies. Examples of each are: Querying Spec Names ~~~~~~~~~~~~~~~~~~~ -If the build has to be customized to the concrete version of an abstract -``Spec`` you can use its ``name`` property. For example: +If the build has to be customized to the concrete version of an abstract ``Spec`` you can use its ``name`` property. +For example: * Is ``openmpi`` the MPI I'm building with? @@ -629,8 +557,7 @@ If the build has to be customized to the concrete version of an abstract Querying Variants ~~~~~~~~~~~~~~~~~ -Adjusting build options based on enabled variants can be done by querying -the ``Spec`` itself, such as: +Adjusting build options based on enabled variants can be done by querying the ``Spec`` itself, such as: * Am I building with the ``debug`` variant? @@ -640,29 +567,23 @@ the ``Spec`` itself, such as: # Add -g option to configure flags -These are just a few examples of ``Spec`` queries. Spack has thousands of -built-in packages that can serve as examples to guide the development -of your package. You can find these packages in -``$SPACK_ROOT/var/spack/repos/builtin/packages``. +These are just a few examples of ``Spec`` queries. +Spack has thousands of built-in packages that can serve as examples to guide the development of your package. +You can find these packages in ``$SPACK_ROOT/var/spack/repos/builtin/packages``. ---------------------- Multiple Build Systems ---------------------- -There are cases where software actively supports two build systems, or changes -build systems as it evolves, or needs different build systems on different platforms. -Spack allows you to write a single, neat recipe for these cases too. It will only -require a slight change in the recipe's structure compared to what we have seen -so far. +There are cases where software actively supports two build systems, or changes build systems as it evolves, or needs different build systems on different platforms. +Spack allows you to write a single, neat recipe for these cases too. +It will only require a slight change in the recipe's structure compared to what we have seen so far. -Let's take ``uncrustify``, a source code beautifier, as an example. This software -used to build with Autotools until version 0.63, and then switched build systems -to CMake at version 0.64. +Let's take ``uncrustify``, a source code beautifier, as an example. +This software used to build with Autotools until version 0.63, and then switched build systems to CMake at version 0.64. -Compared to previous recipes in this tutorial, in this case we need ``Uncrustify`` to -inherit from both ``CMakePackage`` and ``AutotoolsPackage``. We also need to explicitly -specify the ``build_system`` directive, and add conditional dependencies based on -build system: +Compared to previous recipes in this tutorial, in this case we need ``Uncrustify`` to inherit from both ``CMakePackage`` and ``AutotoolsPackage``. +We also need to explicitly specify the ``build_system`` directive, and add conditional dependencies based on build system: .. code-block:: python @@ -684,19 +605,16 @@ build system: with when("build_system=cmake"): depends_on("cmake@3.18:", type="build") -We didn't mention it so far, but each spec has a ``build_system`` variant that specifies -the build system it uses. In most cases that variant has a single allowed value, inherited from the -corresponding base package - so, usually, you don't have to think about it. +We didn't mention it so far, but each spec has a ``build_system`` variant that specifies the build system it uses. +In most cases that variant has a single allowed value, inherited from the corresponding base package - so, usually, you don't have to think about it. -When your package supports more than one build system though, you have to explicitly declare which ones are -allowed and under which conditions. In the example above it's ``cmake`` for version 0.64 and higher and -``autotools`` for version 0.63 and lower. +When your package supports more than one build system though, you have to explicitly declare which ones are allowed and under which conditions. +In the example above it's ``cmake`` for version 0.64 and higher and ``autotools`` for version 0.63 and lower. -The ``build_system`` variant can also be used to declare other properties which are conditional on the build -system being selected. For instance, above we declare that when using ``cmake``, CMake 3.18+ is required. +The ``build_system`` variant can also be used to declare other properties which are conditional on the build system being selected. +For instance, above we declare that when using ``cmake``, CMake 3.18+ is required. -The other relevant difference, compared to the previous recipes we have seen so far, is that the code prescribing -the installation procedure will live into two separate classes: +The other relevant difference, compared to the previous recipes we have seen so far, is that the code prescribing the installation procedure will live into two separate classes: .. code-block:: python @@ -708,15 +626,13 @@ the installation procedure will live into two separate classes: def configure_args(self): pass -Depending on the ``spec``, and more specifically on the value of the ``build_system`` directive, a ``Builder`` -object will be instantiated from one of the two classes when an installation is requested from a user. +Depending on the ``spec``, and more specifically on the value of the ``build_system`` directive, a ``Builder`` object will be instantiated from one of the two classes when an installation is requested from a user. ----------- Cleaning Up ----------- -Before leaving this tutorial, let's ensure what we have done does not -interfere with your Spack instance or future sections of the tutorial. +Before leaving this tutorial, let's ensure what we have done does not interfere with your Spack instance or future sections of the tutorial. Undo the work we've done here by entering the following commands: .. literalinclude:: outputs/packaging/cleanup.out @@ -726,10 +642,8 @@ Undo the work we've done here by entering the following commands: More information -------------------- -This tutorial module only scratches the surface of defining Spack package -recipes. The `Packaging Guide -`_ more -thoroughly covers packaging topics. +This tutorial module only scratches the surface of defining Spack package recipes. +The `Packaging Guide `_ more thoroughly covers packaging topics. Additional information on key topics can be found at the links below. diff --git a/tutorial_scripting.rst b/tutorial_scripting.rst index a3edec8ca4..26a7d2d96b 100644 --- a/tutorial_scripting.rst +++ b/tutorial_scripting.rst @@ -12,76 +12,57 @@ Scripting with Spack ==================== This tutorial introduces advanced Spack features related to scripting. -Specifically, we will show you how to write scripts using ``spack find`` -and ``spack python``. -Earlier sections of the tutorial demonstrated using ``spack find`` to -list and search installed packages. -The ``spack python`` command gives you access to all of Spack's `internal -APIs `_, allowing -you to write more complex queries, for example. +Specifically, we will show you how to write scripts using ``spack find`` and ``spack python``. +Earlier sections of the tutorial demonstrated using ``spack find`` to list and search installed packages. +The ``spack python`` command gives you access to all of Spack's `internal APIs `_, allowing you to write more complex queries, for example. Since Spack has an extensive API, we'll only scratch the surface here. -We'll give you enough information to start writing your own scripts and -to find what you need, with a little digging. +We'll give you enough information to start writing your own scripts and to find what you need, with a little digging. ----------------------------- Scripting with ``spack find`` ----------------------------- -So far, the output we've seen from ``spack find`` has been for human -consumption. But you can take advantage of some advanced options of -the command to generate machine-readable output suitable for piping -to a script. +So far, the output we've seen from ``spack find`` has been for human consumption. +But you can take advantage of some advanced options of the command to generate machine-readable output suitable for piping to a script. ^^^^^^^^^^^^^^^^^^^^^^^ ``spack find --format`` ^^^^^^^^^^^^^^^^^^^^^^^ -The main job of ``spack find`` is to show the user a bunch of concrete -specs that correspond to installed packages. By default, we display them -with some default attributes, like the ``@version`` suffix you're used to -seeing in the output. +The main job of ``spack find`` is to show the user a bunch of concrete specs that correspond to installed packages. +By default, we display them with some default attributes, like the ``@version`` suffix you're used to seeing in the output. -The ``--format`` argument allows you to display the specs however you -choose, using custom format strings. Format strings let you specify the -names of particular *parts* of the specs you want displayed. Let's see -the first option in action. +The ``--format`` argument allows you to display the specs however you choose, using custom format strings. +Format strings let you specify the names of particular *parts* of the specs you want displayed. +Let's see the first option in action. -Suppose you only want to display the *name*, *version*, and first ten (10) -characters of the *hash* for every package installed in your Spack -instance. You can generate that output with the following command: +Suppose you only want to display the *name*, *version*, and first ten (10) characters of the *hash* for every package installed in your Spack instance. +You can generate that output with the following command: .. literalinclude:: outputs/scripting/find-format.out :language: console :emphasize-lines: 1 -Note that ``name``, ``version``, and ``hash`` are attributes of Spack's -internal ``Spec`` object and enclosing them in braces ensures they are -output according to your format string. +Note that ``name``, ``version``, and ``hash`` are attributes of Spack's internal ``Spec`` object and enclosing them in braces ensures they are output according to your format string. -Using ``spack find --format`` allows you to retrieve just the information -you need to do things like pipe the output to typical UNIX command-line -tools like ``sort`` or ``uniq``. +Using ``spack find --format`` allows you to retrieve just the information you need to do things like pipe the output to typical UNIX command-line tools like ``sort`` or ``uniq``. ^^^^^^^^^^^^^^^^^^^^^ ``spack find --json`` ^^^^^^^^^^^^^^^^^^^^^ -Alternatively, you can get a serialized version of Spec objects in -the `JSON` format using the ``--json`` option. For example, you can -get attributes for all installations of ``zlib-ng`` by entering: +Alternatively, you can get a serialized version of Spec objects in the `JSON` format using the ``--json`` option. +For example, you can get attributes for all installations of ``zlib-ng`` by entering: .. literalinclude:: outputs/scripting/find-json.out :language: console :emphasize-lines: 1 -The ``spack find --json`` command gives you everything we know about -the specs in a structured format. You can pipe its output to -JSON filtering tools like ``jq`` to extract just the parts you want. +The ``spack find --json`` command gives you everything we know about the specs in a structured format. +You can pipe its output to JSON filtering tools like ``jq`` to extract just the parts you want. -Check out the `basic usage docs -`_ -for more examples. +Check out the `basic usage docs `_ for more examples. ---------------------------------------- @@ -90,9 +71,9 @@ Introducing the ``spack python`` command What if we need to perform more advanced queries? -Spack provides the ``spack python`` command to launch a python interpreter -with Spack's python modules available to import. It uses the underlying -python for the rest of its commands. So you can write scripts to: +Spack provides the ``spack python`` command to launch a python interpreter with Spack's python modules available to import. +It uses the underlying python for the rest of its commands. +So you can write scripts to: - run Spack commands; - explore abstract and concretized specs; and @@ -104,37 +85,32 @@ Let's launch a Spack-aware python interpreter by entering: :language: console :emphasize-lines: 1,5 -Since we are in a python interpreter, use ``exit()`` to end -the session and return to the terminal. +Since we are in a python interpreter, use ``exit()`` to end the session and return to the terminal. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Accessing the ``Spec`` object ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Now let's take a look at the internal representation of the Spack ``Spec``. -As you already know, specs can be either *abstract* or *concrete*. The -specs you've seen in ``package.py`` files (e.g., in the ``install()`` -method) have been *concrete*, or fully specified. The specs you've typed -on the command line have been *abstract*. Understanding the differences -between the two types is key to using Spack's internal API. +As you already know, specs can be either *abstract* or *concrete*. +The specs you've seen in ``package.py`` files (e.g., in the ``install()`` method) have been *concrete*, or fully specified. +The specs you've typed on the command line have been *abstract*. +Understanding the differences between the two types is key to using Spack's internal API. -Let's open another python interpreter with ``spack python``, instantiate -the ``zlib`` spec, and check a few properties of an abstract spec: +Let's open another python interpreter with ``spack python``, instantiate the ``zlib`` spec, and check a few properties of an abstract spec: .. literalinclude:: outputs/scripting/spack-python-abstract.out :language: console :emphasize-lines: 1-3,5,11,13 -Notice that there are ``Spec`` properties and methods that are not -accessible to abstract specs; specifically: +Notice that there are ``Spec`` properties and methods that are not accessible to abstract specs; specifically: - an exception -- ``SpecError`` -- is raised if we try to access its ``version``; - there are no associated ``versions``; and - the spec's operating system is ``None``. -Now, without exiting the interpreter, let's concretize the spec and try -again: +Now, without exiting the interpreter, let's concretize the spec and try again: .. literalinclude:: outputs/scripting/spack-python-concrete.out :language: console @@ -146,8 +122,7 @@ Notice that the concretized spec now: - has a single entry in its ``versions`` list; and - the operating system is now ``ubuntu22.04``. -It is not necessary to store the intermediate abstract spec -- you can -use the ``.concretized()`` method as shorthand: +It is not necessary to store the intermediate abstract spec -- you can use the ``.concretized()`` method as shorthand: .. literalinclude:: outputs/scripting/spack-python-sans-intermediate.out :language: console @@ -157,11 +132,10 @@ use the ``.concretized()`` method as shorthand: Querying the Spack database ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Even more powerful queries are available when we look at the information -stored in the Spack database. The ``Database`` object in Spack is in the -``spack.store.STORE.db`` variable. We'll interact with it mainly through the -``query()`` method. Let's see the documentation available for ``query()`` -using python's built-in ``help()`` function: +Even more powerful queries are available when we look at the information stored in the Spack database. +The ``Database`` object in Spack is in the ``spack.store.STORE.db`` variable. +We'll interact with it mainly through the ``query()`` method. +Let's see the documentation available for ``query()`` using python's built-in ``help()`` function: .. literalinclude:: outputs/scripting/spack-python-db-query-help.out :language: console @@ -169,17 +143,13 @@ using python's built-in ``help()`` function: We will primarily make use of the ``query_spec`` argument. -Recall that queries using the ``spack find`` command are limited to -queries of attributes with matching values, not values they do *not* -have. In other words, we cannot use the ``spack find`` command for -all packages that *do not* satisfy a certain criterion. +Recall that queries using the ``spack find`` command are limited to queries of attributes with matching values, not values they do *not* have. +In other words, we cannot use the ``spack find`` command for all packages that *do not* satisfy a certain criterion. We *can* use the python interface to write these types of queries. -For example, let's find all packages that were compiled with ``gcc`` but -do not depend on ``mpich``. We can do this by using custom python code -and Spack database queries. We will use the ``spack.cmd.display_specs`` -for output to achieve the same printing functionality as the ``spack -find`` command: +For example, let's find all packages that were compiled with ``gcc`` but do not depend on ``mpich``. +We can do this by using custom python code and Spack database queries. +We will use the ``spack.cmd.display_specs`` for output to achieve the same printing functionality as the ``spack find`` command: .. literalinclude:: outputs/scripting/spack-python-db-query-exclude.out :language: console @@ -199,18 +169,15 @@ before generalizing the functionality for reuse. Using scripts ^^^^^^^^^^^^^ -Now let's parameterize our script to accept arguments on the command -line. With a few generalizations to use the include and exclude specs -as arguments, we can create a powerful, general-purpose query script. +Now let's parameterize our script to accept arguments on the command line. +With a few generalizations to use the include and exclude specs as arguments, we can create a powerful, general-purpose query script. -Open a file called ``find_exclude.py`` in your preferred editor -and add the following code: +Open a file called ``find_exclude.py`` in your preferred editor and add the following code: .. literalinclude:: outputs/scripting/0.find_exclude.py.example :language: python -Notice we added importing and using the system package (``sys``) -to access the first and second command line arguments. +Notice we added importing and using the system package (``sys``) to access the first and second command line arguments. Now we can run our new script by entering the following: @@ -218,43 +185,33 @@ Now we can run our new script by entering the following: :language: console :emphasize-lines: 1 -This is *great* for us, as long as we remember to use Spack's -``python`` command to run it. +This is *great* for us, as long as we remember to use Spack's ``python`` command to run it. ------------------------------------- Using the ``spack-python`` executable ------------------------------------- -What if we want to make our script available for others to use without -the hassle of having to remember to use ``spack python``? +What if we want to make our script available for others to use without the hassle of having to remember to use ``spack python``? -We can take advantage of the shebang line typically added as the -first line of python executable files. But there is a catch, as -we will soon see. +We can take advantage of the shebang line typically added as the first line of python executable files. +But there is a catch, as we will soon see. -Open the ``find_exclude.py`` script we created above in your preferred -editor and add the shebang line with ``spack python`` as the arguments -to ``env``: +Open the ``find_exclude.py`` script we created above in your preferred editor and add the shebang line with ``spack python`` as the arguments to ``env``: .. literalinclude:: outputs/scripting/1.find_exclude.py.example :language: python :emphasize-lines: 1 -Then exit our editor and add execute permissions to the script before -running it as follows: +Then exit our editor and add execute permissions to the script before running it as follows: .. literalinclude:: outputs/scripting/find-exclude-2.out :language: console :emphasize-lines: 1-2 If you are lucky, it worked on your system, but there is no guarantee. -Some systems only support a single argument on the shebang line (see -`here `_). -``spack-python``, which is a wrapper script for ``spack python``, solves -this issue. +Some systems only support a single argument on the shebang line (see `here `_). ``spack-python``, which is a wrapper script for ``spack python``, solves this issue. -Bring up the file in your editor again and change the ``env`` argument -to ``spack-python`` as follows: +Bring up the file in your editor again and change the ``env`` argument to ``spack-python`` as follows: .. literalinclude:: outputs/scripting/2.find_exclude.py.example :language: python @@ -268,8 +225,8 @@ Exit your editor and let's run the script again: Congratulations! It will now work on any system with Spack installed. -You now have the basic tools to create your own custom Spack queries and -prototype ideas. We hope one day you'll contribute them back to Spack. +You now have the basic tools to create your own custom Spack queries and prototype ideas. +We hope one day you'll contribute them back to Spack. .. LocalWords: LLC Spack's APIs hdf zlib literalinclude json uniq jq .. LocalWords: docs concretized REPL API SpecError spec's py ubuntu diff --git a/tutorial_stacks.rst b/tutorial_stacks.rst index 560fd67b99..1f2516a134 100644 --- a/tutorial_stacks.rst +++ b/tutorial_stacks.rst @@ -12,23 +12,17 @@ Stacks Tutorial =============== So far, we've talked about Spack environments in the context of a unified user environment. -But environments in Spack have much broader capabilities. In this tutorial we will consider -how to use Spack environments to manage large deployments of software. +But environments in Spack have much broader capabilities. +In this tutorial we will consider how to use Spack environments to manage large deployments of software. -What usually differs between a typical environment for a single user, and an environment used -to manage large deployments, is that in the latter case we often have a set of packages we want -to install across a wide range of MPIs, LAPACKs or compilers. +What usually differs between a typical environment for a single user, and an environment used to manage large deployments, is that in the latter case we often have a set of packages we want to install across a wide range of MPIs, LAPACKs or compilers. -In the following we'll mimic the creation of a software stack built onto a cross-product of -different LAPACK and MPI libraries, with a compiler that is more recent than the one provided -by the host system. +In the following we'll mimic the creation of a software stack built onto a cross-product of different LAPACK and MPI libraries, with a compiler that is more recent than the one provided by the host system. In the first part we'll focus on how to properly configure and install the software we want. -We'll learn how to pin certain requirements, and how to write a cross product of specs in a -compact, and expressive, way. +We'll learn how to pin certain requirements, and how to write a cross product of specs in a compact, and expressive, way. -Then we'll consider how the software we install might be consumed by our users, and see the -two main mechanisms that Spack provides for that: views and module files. +Then we'll consider how the software we install might be consumed by our users, and see the two main mechanisms that Spack provides for that: views and module files. .. note:: @@ -49,16 +43,16 @@ This is, currently, an iterative process that can be done in two ways: 1. Install the compiler first, then register it in the environment 2. Use a second environment just for the compiler -In the following we'll use the first approach. For people interested, an example -of the latter approach can be found `at this link `_. +In the following we'll use the first approach. +For people interested, an example of the latter approach can be found `at this link `_. Let's start by creating an environment in a directory of our choice: .. literalinclude:: outputs/stacks/setup-0.out :language: console -Now we can add from the command line a new compiler. We'll also disable the generation of views -for the time being, as we'll come back to this topic later in the tutorial: +Now we can add from the command line a new compiler. +We'll also disable the generation of views for the time being, as we'll come back to this topic later in the tutorial: .. literalinclude:: outputs/stacks/setup-1.out :language: console @@ -79,15 +73,13 @@ Finally, let's register it as a new compiler in the environment: .. literalinclude:: outputs/stacks/compiler-find-0.out :language: console -The ``spack location -i`` command returns the installation -prefix for the spec being queried: +The ``spack location -i`` command returns the installation prefix for the spec being queried: .. literalinclude:: outputs/stacks/compiler-find-1.out :language: console -This might be useful in general when scripting Spack commands, as the -example above shows. Listing the compilers now shows the presence -of ``gcc@12.3.0``: +This might be useful in general when scripting Spack commands, as the example above shows. +Listing the compilers now shows the presence of ``gcc@12.3.0``: .. literalinclude:: outputs/stacks/compiler-list-0.out :language: console @@ -104,8 +96,7 @@ Install a software stack ------------------------ Now that we have a compiler ready, the next objective is to build software with it. -We'll start by trying to add different versions of ``netlib-scalapack``, linked against -different MPI implementations: +We'll start by trying to add different versions of ``netlib-scalapack``, linked against different MPI implementations: .. literalinclude:: outputs/stacks/unify-0.out :language: console @@ -115,8 +106,7 @@ If we try to concretize the environment, we'll get an error: .. literalinclude:: outputs/stacks/unify-1.out :language: console -The error message is quite verbose, and admittedly complicated, but at the end it gives -a useful hint: +The error message is quite verbose, and admittedly complicated, but at the end it gives a useful hint: .. code-block:: @@ -128,17 +118,16 @@ Let's see what that means. Tuning concretizer options for a stack ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Whenever we concretize an environment with more than one root spec, we can configure Spack to be more -or less strict with duplicate nodes in the sub-DAG obtained by following link and run edges starting -from the roots. We usually call this sub-DAG the *root unification set*. +Whenever we concretize an environment with more than one root spec, we can configure Spack to be more or less strict with duplicate nodes in the sub-DAG obtained by following link and run edges starting from the roots. +We usually call this sub-DAG the *root unification set*. A diagram might help to better visualize the concept: .. image:: _static/images/stacks-unify.svg -The image above represents the current environment, with our three root specs highlighted by a thicker -dashed line. Any node that could be reached following a link or run edge is part of the root unification -set. Pure build dependencies might fall outside of it. +The image above represents the current environment, with our three root specs highlighted by a thicker dashed line. +Any node that could be reached following a link or run edge is part of the root unification set. +Pure build dependencies might fall outside of it. The config option determining which nodes are allowed to be in the root unification set is ``concretizer:unify``. Let's check its value: @@ -146,18 +135,18 @@ Let's check its value: .. literalinclude:: outputs/stacks/unify-2.out :language: console -``concretizer:unify:true`` means that only a single configuration for each package can be present. This value -is good for single project environments, since it ensures we can construct a view of all the software, with the -usual structure expected on a Unix-ish system, and without risks of collisions between installations. +``concretizer:unify:true`` means that only a single configuration for each package can be present. +This value is good for single project environments, since it ensures we can construct a view of all the software, with the usual structure expected on a Unix-ish system, and without risks of collisions between installations. -Clearly, we can't respect this requirement, since our roots already contain two different configurations of -``netlib-scalapack``. Let's set the value to ``false``, and try to re-concretize: +Clearly, we can't respect this requirement, since our roots already contain two different configurations of ``netlib-scalapack``. +Let's set the value to ``false``, and try to re-concretize: .. literalinclude:: outputs/stacks/unify-3.out :language: console -This time concretization succeeded. Setting ``concretizer:unify:false`` is effectively concretizing each root -spec on its own, and then merging the results into the environment. This allows us to have the duplicates we need. +This time concretization succeeded. +Setting ``concretizer:unify:false`` is effectively concretizing each root spec on its own, and then merging the results into the environment. +This allows us to have the duplicates we need. .. note:: @@ -167,22 +156,21 @@ spec on its own, and then merging the results into the environment. This allows $ spack config add concretizer:unify:when_possible - With this option Spack will try to unify the environment in an eager way, solving it in multiple rounds. - The concretization at round ``n`` will contain all the specs that could not be unified at round ``n-1``, - and will consider all the specs from previous rounds for reuse. +With this option Spack will try to unify the environment in an eager way, solving it in multiple rounds. +The concretization at round ``n`` will contain all the specs that could not be unified at round ``n-1``, and will consider all the specs from previous rounds for reuse. ^^^^^^^^^^^^^ Spec matrices ^^^^^^^^^^^^^ -Let's expand our stack further and consider also linking against different LAPACK providers. We could, of -course, add new specs explicitly: +Let's expand our stack further and consider also linking against different LAPACK providers. +We could, of course, add new specs explicitly: .. literalinclude:: outputs/stacks/unify-4.out :language: console -This way of proceeding, though, will become very tedious as soon as more software is requested. The best -way to express a cross-product like this in Spack is instead through a matrix: +This way of proceeding, though, will become very tedious as soon as more software is requested. +The best way to express a cross-product like this in Spack is instead through a matrix: .. literalinclude:: outputs/stacks/examples/2.spack.stack.yaml :language: yaml @@ -223,14 +211,10 @@ As we can see we have our four variations of ``netlib-scalapack`` installed. Reusable definitions ^^^^^^^^^^^^^^^^^^^^ -So far, we have seen how we can use spec matrices to generate cross-product specs -from rows containing a list of constraints. A common situation you will encounter -with large deployments is the necessity to add multiple matrices to the list of specs, -that possibly share some of those rows. +So far, we have seen how we can use spec matrices to generate cross-product specs from rows containing a list of constraints. +A common situation you will encounter with large deployments is the necessity to add multiple matrices to the list of specs, that possibly share some of those rows. -To reduce the amount of duplication needed in the manifest file, and thus the maintenance -burden for people maintaining it, Spack allows to *define* lists of constraints under -the ``definitions`` attribute, and expand them later when needed. +To reduce the amount of duplication needed in the manifest file, and thus the maintenance burden for people maintaining it, Spack allows to *define* lists of constraints under the ``definitions`` attribute, and expand them later when needed. Let's rewrite our manifest in that sense: .. literalinclude:: outputs/stacks/examples/3.spack.stack.yaml @@ -246,8 +230,8 @@ Now we can use those definitions to add e.g. serial packages built against the L Let's try to do that by using ``py-scypy`` as an example: Another ability that is often useful, is that of excluding specific entries from a cross-product matrix. -We can do that with the ``exclude`` keyword, in the same item as the ``matrix``. Let's try to remove -``py-scipy ^netlib-lapack`` from our matrix: +We can do that with the ``exclude`` keyword, in the same item as the ``matrix``. +Let's try to remove ``py-scipy ^netlib-lapack`` from our matrix: .. literalinclude:: outputs/stacks/examples/4bis.spack.stack.yaml :language: yaml @@ -267,9 +251,9 @@ At this point the environment contains only ``py-scipy ^openblas``. Let's verify Conditional definitions ^^^^^^^^^^^^^^^^^^^^^^^ -Spec list definitions can also be conditioned on a ``when`` clause. The ``when`` clause -is a python conditional that is evaluated in a restricted environment. The variables -available in ``when`` clauses are: +Spec list definitions can also be conditioned on a ``when`` clause. +The ``when`` clause is a python conditional that is evaluated in a restricted environment. +The variables available in ``when`` clauses are: ================= =========== variable name value @@ -284,23 +268,22 @@ variable name value ``hostname`` The hostname of this node ================= =========== -Let's say we only want to limit to just use ``mpich``, unless the ``SPACK_STACK_USE_OPENMPI`` -environment variable is set. To do so we could write the following ``spack.yaml``: +Let's say we only want to limit to just use ``mpich``, unless the ``SPACK_STACK_USE_OPENMPI`` environment variable is set. +To do so we could write the following ``spack.yaml``: .. literalinclude:: outputs/stacks/examples/5.spack.stack.yaml :language: yaml :emphasize-lines: 7-9 -Different definitions of lists with the same name are concatenated, so we can define our MPI list -in one place unconditionally, and then conditionally append one or more values to it. +Different definitions of lists with the same name are concatenated, so we can define our MPI list in one place unconditionally, and then conditionally append one or more values to it. Let's first check what happens when we concretize and don't set any environment variable: .. literalinclude:: outputs/stacks/concretize-5.out :language: console -As we expected now we are only using ``mpich`` as an MPI provider. To get ``openmpi`` back -we just need to set the appropriate environment variable: +As we expected now we are only using ``mpich`` as an MPI provider. +To get ``openmpi`` back we just need to set the appropriate environment variable: .. literalinclude:: outputs/stacks/concretize-6.out :language: console @@ -311,15 +294,15 @@ There is no need to install this time, since all the specs were still in the sto Other useful features ^^^^^^^^^^^^^^^^^^^^^ -Sometimes it might be useful to create a local source mirror for the specs installed in an environment. If the -environment is active, this is as simple as: +Sometimes it might be useful to create a local source mirror for the specs installed in an environment. +If the environment is active, this is as simple as: .. code-block:: console $ spack mirror create --all -d ./stacks-mirror -This command fetches all the tarballs for the packages in the ``spack.lock`` file, and puts them in the directory -passed as argument. Later you can move this mirror to e.g. an air-gapped machine and: +This command fetches all the tarballs for the packages in the ``spack.lock`` file, and puts them in the directory passed as argument. +Later you can move this mirror to e.g. an air-gapped machine and: .. code-block:: console @@ -332,13 +315,11 @@ to be able to re-build the specs from sources. If instead you want to create a b $ spack gpg create $ spack buildcache push ./mirror -In that case, don't forget to set an appropriate value for the padding of the install tree, -see `how to setup relocation `_ -in our documentation. +In that case, don't forget to set an appropriate value for the padding of the install tree, see `how to setup relocation `_ in our documentation. -By default, Spack installs one package at a time, using the ``-j`` option where it can. If you are installing a large -environment, and have at disposal a beefy build node, you might need to start more installations in parallel to make an -optimal use of the resources. This can be done by creating a ``depfile``, when the environment is active: +By default, Spack installs one package at a time, using the ``-j`` option where it can. +If you are installing a large environment, and have at disposal a beefy build node, you might need to start more installations in parallel to make an optimal use of the resources. +This can be done by creating a ``depfile``, when the environment is active: .. code-block:: console @@ -353,43 +334,40 @@ Make the software stack easy to use ----------------------------------- Now that the software stack has been installed, we need to focus on how it can be used by our customers. -We'll first see how we can configure views to project a subset of the specs we installed onto a -filesystem folder with the usual Unix structure. Then we'll have a similar discussion for module files. +We'll first see how we can configure views to project a subset of the specs we installed onto a filesystem folder with the usual Unix structure. +Then we'll have a similar discussion for module files. Which of the two approaches is better depends strongly on the use case at hand. ^^^^^^^^^^^^^^^^ View descriptors ^^^^^^^^^^^^^^^^ -At the beginning, we configured Spack not to create a view for this stack because simple views won't work -with stacks. We've been concretizing multiple packages of the same name, and they would conflict if linked -into the same view. +At the beginning, we configured Spack not to create a view for this stack because simple views won't work with stacks. +We've been concretizing multiple packages of the same name, and they would conflict if linked into the same view. -What we can do is create *multiple views*, using view descriptors. This would allows us to define which -packages are linked into the view, and how. Let's edit our ``spack.yaml`` file again. +What we can do is create *multiple views*, using view descriptors. +This would allows us to define which packages are linked into the view, and how. +Let's edit our ``spack.yaml`` file again. .. literalinclude:: outputs/stacks/examples/6.spack.stack.yaml :language: yaml :emphasize-lines: 44-54 In the configuration above we created two views, named ``default`` and ``full``. -The ``default`` view consists of all the packages that are compiled with ``gcc@12``, but do not depend on -either ``mpich`` or ``netlib-lapack``. As we can see, we can both *include* and *exclude* specs using -constrains. +The ``default`` view consists of all the packages that are compiled with ``gcc@12``, but do not depend on either ``mpich`` or ``netlib-lapack``. +As we can see, we can both *include* and *exclude* specs using constrains. -The ``full`` view contains a more complex projection, so to put each spec into an appropriate -subdirectory, according to the first constraint that the spec matches. ``all`` is the default -projection, and has always the lowest priority, independent of the order in which it appears. To avoid -confusion, we advise to always keep it last in projections. +The ``full`` view contains a more complex projection, so to put each spec into an appropriate subdirectory, according to the first constraint that the spec matches. ``all`` is the default projection, and has always the lowest priority, independent of the order in which it appears. +To avoid confusion, we advise to always keep it last in projections. Let's concretize to regenerate the views, and check their structure: .. literalinclude:: outputs/stacks/view-0.out :language: console -The view descriptor also contains a ``link`` key. The default behavior, as we have seen, is to link all -packages, including implicit link and run dependencies, into the view. If we set the option to "roots", -Spack links only the root packages into the view. +The view descriptor also contains a ``link`` key. +The default behavior, as we have seen, is to link all packages, including implicit link and run dependencies, into the view. +If we set the option to "roots", Spack links only the root packages into the view. .. literalinclude:: outputs/stacks/examples/7.spack.stack.yaml :language: yaml @@ -398,19 +376,19 @@ Spack links only the root packages into the view. .. literalinclude:: outputs/stacks/view-1.out :language: console -Now we see only the root libraries in the default view. The rest are hidden, but are still available in the full view. +Now we see only the root libraries in the default view. +The rest are hidden, but are still available in the full view. The complete documentation on view can be found `here `_. ^^^^^^^^^^^^ Module files ^^^^^^^^^^^^ -Module files are another very popular way to use software on HPC systems. In this section -we'll show how to configure and generate a hierarchical module structure, suitable for ``lmod``. +Module files are another very popular way to use software on HPC systems. +In this section we'll show how to configure and generate a hierarchical module structure, suitable for ``lmod``. A more in-depth tutorial, focused only on module files, can be found at :ref:`modules-tutorial`. -There we discuss the general architecture of module file generation in Spack and we highlight -differences between ``environment-modules`` and ``lmod`` that won't be covered in this section. +There we discuss the general architecture of module file generation in Spack and we highlight differences between ``environment-modules`` and ``lmod`` that won't be covered in this section. So, let's start by adding ``lmod`` to the software installed with the system compiler: @@ -437,8 +415,7 @@ The next step is to add some basic configuration to our ``spack.yaml`` to genera :language: yaml :emphasize-lines: 45-54 -In these few lines of additional configuration we told Spack to generate ``lmod`` module files -in a subdirectory named ``modules``, using a hierarchy comprising both ``lapack`` and ``mpi``. +In these few lines of additional configuration we told Spack to generate ``lmod`` module files in a subdirectory named ``modules``, using a hierarchy comprising both ``lapack`` and ``mpi``. We can generate the module files and use them with the following commands: @@ -452,20 +429,17 @@ Now we should be able to see the module files that have been generated: .. literalinclude:: outputs/stacks/modules-2.out :language: console -The sets of modules is already usable, and the hierarchy already works. For instance we can -load the ``gcc`` compiler and check that we have ``gcc`` in out path and we have a lot more -modules available - all the ones compiled with ``gcc@12.3.0``: +The sets of modules is already usable, and the hierarchy already works. +For instance we can load the ``gcc`` compiler and check that we have ``gcc`` in out path and we have a lot more modules available - all the ones compiled with ``gcc@12.3.0``: .. literalinclude:: outputs/stacks/modules-3.out :language: console -There are a few issues though. For once, we have a lot of modules generated from dependencies -of ``gcc`` that are cluttering the view, and won't likely be needed directly by users. Then, module -names contain hashes, which go against users being able to reuse the same script in similar, but -not equal, environments. +There are a few issues though. +For once, we have a lot of modules generated from dependencies of ``gcc`` that are cluttering the view, and won't likely be needed directly by users. +Then, module names contain hashes, which go against users being able to reuse the same script in similar, but not equal, environments. -Also, some of the modules might need to set custom environment variables, which are specific to -the deployment aspects that don't enter the hash - for instance a policy at the deploying site. +Also, some of the modules might need to set custom environment variables, which are specific to the deployment aspects that don't enter the hash - for instance a policy at the deploying site. To address all these needs we can complicate out ``modules`` configuration a bit more: @@ -489,7 +463,6 @@ This concludes the quick tour of module file generation, and the tutorial on sta Summary ------- -In this tutorial, we configured Spack to install a stack of software built on a cross-product of different -MPI and LAPACK libraries. We used the spec matrix syntax to express in a compact way the specs to be installed, -and spec list definitions to reuse the same matrix rows in different places. Then, we discussed how to make the -software easy to use, leveraging either filesystem views or module files. +In this tutorial, we configured Spack to install a stack of software built on a cross-product of different MPI and LAPACK libraries. +We used the spec matrix syntax to express in a compact way the specs to be installed, and spec list definitions to reuse the same matrix rows in different places. +Then, we discussed how to make the software easy to use, leveraging either filesystem views or module files.