|
| 1 | +#!/usr/bin/python3 |
| 2 | + |
| 3 | +import argparse |
| 4 | +import datetime |
| 5 | +import json |
| 6 | +import os |
| 7 | +import subprocess |
| 8 | +import tempfile |
| 9 | +import shutil |
| 10 | +from stat import ( |
| 11 | + S_IREAD, |
| 12 | + S_IRGRP, |
| 13 | + S_IROTH) |
| 14 | +from cosalib.builds import Builds |
| 15 | +from cosalib.cmdlib import ( |
| 16 | + rfc3339_time, |
| 17 | + get_basearch, |
| 18 | + sha256sum_file) |
| 19 | + |
| 20 | +VALID_NAMES = ["fedora-coreos", "rhcos", "scos"] |
| 21 | + |
| 22 | + |
| 23 | +def main(): |
| 24 | + args = parse_args() |
| 25 | + with tempfile.TemporaryDirectory(prefix='cosa-import-', dir='tmp') as tmpd: |
| 26 | + tmp_ociarchive = os.path.join(tmpd, "out.ociarchive") |
| 27 | + |
| 28 | + metadata = prepare_build(args, tmp_ociarchive) |
| 29 | + |
| 30 | + name = metadata['Labels']['com.coreos.osname'] |
| 31 | + buildid = metadata['Labels']['org.opencontainers.image.version'] |
| 32 | + |
| 33 | + builds = Builds() |
| 34 | + if builds.has(buildid): |
| 35 | + raise Exception(f"Build ID {buildid} already exists!") |
| 36 | + |
| 37 | + arch = get_basearch() |
| 38 | + |
| 39 | + manifest = generate_manifest_json(tmpd, tmp_ociarchive, name, buildid, arch) |
| 40 | + meta_json = generate_meta_json(tmp_ociarchive, metadata, manifest['metadata'], name) |
| 41 | + |
| 42 | + # import into the tmp/repo not only so it's cached, but also to get the ostree-commit |
| 43 | + meta_json['ostree-commit'] = ostree_import(tmpd, tmp_ociarchive, buildid) |
| 44 | + |
| 45 | + finalize_build(builds, tmp_ociarchive, meta_json, manifest, buildid, arch) |
| 46 | + |
| 47 | + |
| 48 | +def parse_args(): |
| 49 | + parser = argparse.ArgumentParser(prog='cosa import') |
| 50 | + parser.add_argument("srcimg", metavar='IMAGE', |
| 51 | + help="image to import (containers-transports(5) format)") |
| 52 | + return parser.parse_args() |
| 53 | + |
| 54 | + |
| 55 | +def prepare_build(args, target_ociarchive): |
| 56 | + import_oci_archive(args, target_ociarchive) |
| 57 | + return inspect_oci_archive(target_ociarchive) |
| 58 | + |
| 59 | + |
| 60 | +def finalize_build(builds, source_ociarchive, meta_json, manifest, buildid, arch): |
| 61 | + os.makedirs(f'builds/{buildid}/{arch}/', exist_ok=True) |
| 62 | + |
| 63 | + archive_name = meta_json['images']['ostree']['path'] |
| 64 | + archive_path = f'builds/{buildid}/{arch}/{archive_name}' |
| 65 | + # Move ociarchive to build dir |
| 66 | + shutil.move(source_ociarchive, archive_path) |
| 67 | + |
| 68 | + # move manifest file |
| 69 | + manifest_fname = manifest['metadata']['path'] |
| 70 | + shutil.move(manifest['src_path'], f'builds/{buildid}/{arch}/{manifest_fname}') |
| 71 | + |
| 72 | + with open(f'builds/{buildid}/{arch}/meta.json', 'w') as meta_file: |
| 73 | + json.dump(meta_json, meta_file, indent=4) |
| 74 | + |
| 75 | + # Symlink build to latest |
| 76 | + if os.path.exists('builds/latest'): |
| 77 | + os.remove('builds/latest') |
| 78 | + os.symlink(f'{buildid}', 'builds/latest', target_is_directory=True) |
| 79 | + |
| 80 | + update_builds_json(builds, buildid, arch) |
| 81 | + |
| 82 | + print(f'Imported OCI image as build {buildid}') |
| 83 | + |
| 84 | + |
| 85 | +def update_builds_json(builds, buildid, arch): |
| 86 | + builds.insert_build(buildid, arch) |
| 87 | + builds.bump_timestamp() |
| 88 | + builds.flush() |
| 89 | + |
| 90 | + |
| 91 | +def import_oci_archive(args, target): |
| 92 | + subprocess.check_call(['skopeo', 'copy', args.srcimg, |
| 93 | + f"oci-archive:{target}"]) |
| 94 | + |
| 95 | + |
| 96 | +def inspect_oci_archive(image): |
| 97 | + out = subprocess.check_output(['skopeo', 'inspect', |
| 98 | + f'oci-archive:{image}']) |
| 99 | + return json.loads(out) |
| 100 | + |
| 101 | + |
| 102 | +def generate_manifest_json(tmpd, ociarchive, name, buildid, arch): |
| 103 | + manifest = subprocess.check_output(["skopeo", "inspect", "--raw", f"oci-archive:{ociarchive}"]) |
| 104 | + |
| 105 | + ostree_oci_manifest_path = f"{name}-{buildid}-ostree.{arch}-manifest.json" |
| 106 | + manifest_json_dest = f'{tmpd}/manifest.json' |
| 107 | + |
| 108 | + manifest_json_sha256 = None |
| 109 | + manifest_json_size = None |
| 110 | + with open(manifest_json_dest, 'wb') as manifest_json: |
| 111 | + manifest_json.write(manifest) |
| 112 | + os.fchmod(manifest_json.fileno(), S_IREAD | S_IRGRP | S_IROTH) |
| 113 | + |
| 114 | + manifest_json_sha256 = sha256sum_file(manifest_json_dest) |
| 115 | + manifest_json_size = os.path.getsize(manifest_json_dest) |
| 116 | + |
| 117 | + manifest_metadata = { |
| 118 | + 'path': ostree_oci_manifest_path, |
| 119 | + 'sha256': manifest_json_sha256, |
| 120 | + 'size': manifest_json_size, |
| 121 | + "skip-compression": True, |
| 122 | + } |
| 123 | + |
| 124 | + return { |
| 125 | + 'metadata': manifest_metadata, |
| 126 | + 'src_path': manifest_json_dest |
| 127 | + } |
| 128 | + |
| 129 | + |
| 130 | +def parse_timestamp(timestamp): |
| 131 | + # datetime's doesn't support nanoseconds. |
| 132 | + # So trim it. |
| 133 | + if len(timestamp) > 26 and timestamp[19] == '.': |
| 134 | + timestamp = timestamp[:26] + "Z" |
| 135 | + |
| 136 | + timestamp = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ') |
| 137 | + return rfc3339_time(timestamp.replace(tzinfo=datetime.timezone.utc)) |
| 138 | + |
| 139 | + |
| 140 | +def generate_meta_json(ociarchive, metadata, oci_manifest, name): |
| 141 | + archive_sha256sum = sha256sum_file(ociarchive) |
| 142 | + |
| 143 | + # let raise if missing |
| 144 | + assert metadata['Labels']['containers.bootc'] == '1' |
| 145 | + |
| 146 | + buildid = metadata['Labels']['org.opencontainers.image.version'] |
| 147 | + arch = get_basearch() |
| 148 | + created_timestamp = parse_timestamp(metadata['Created']) |
| 149 | + |
| 150 | + meta_json = { |
| 151 | + 'ostree-version': buildid, # proxy version label |
| 152 | + 'buildid': buildid, # also version label |
| 153 | + 'coreos-assembler.build-timestamp': created_timestamp, # proxy OCI build timestamp |
| 154 | + 'coreos-assembler.oci-imported': True, |
| 155 | + 'name': name, |
| 156 | + 'ostree-timestamp': created_timestamp, |
| 157 | + 'images': { |
| 158 | + 'ostree': { |
| 159 | + "path": f"{name}-{buildid}-ostree.{arch}.ociarchive", |
| 160 | + "sha256": archive_sha256sum, |
| 161 | + "skip-compression": True |
| 162 | + }, |
| 163 | + 'oci-manifest': oci_manifest, |
| 164 | + }, |
| 165 | + 'coreos-assembler.basearch': arch, |
| 166 | + } |
| 167 | + |
| 168 | + return meta_json |
| 169 | + |
| 170 | + |
| 171 | +def ostree_import(parent_tmpd, ociarchive, buildid): |
| 172 | + with tempfile.TemporaryDirectory(dir=parent_tmpd) as tmpd: |
| 173 | + subprocess.check_call(['ostree', 'init', '--repo', tmpd, '--mode=bare-user']) |
| 174 | + |
| 175 | + # import all the blob refs for more efficient import into bare-user repo |
| 176 | + blob_refs = subprocess.check_output(['ostree', 'refs', '--repo', 'tmp/repo', |
| 177 | + '--list', 'ostree/container/blob'], |
| 178 | + encoding='utf-8').splitlines() |
| 179 | + subprocess.check_call(['ostree', 'pull-local', '--repo', tmpd, 'tmp/repo'] + blob_refs) |
| 180 | + |
| 181 | + subprocess.check_call(['ostree', 'container', 'image', 'pull', tmpd, |
| 182 | + f'ostree-unverified-image:oci-archive:{ociarchive}']) |
| 183 | + |
| 184 | + # awkwardly work around the fact that there is no --write-ref equivalent |
| 185 | + refs = subprocess.check_output(['ostree', 'refs', '--repo', tmpd, |
| 186 | + '--list', 'ostree/container/image'], |
| 187 | + encoding='utf-8').splitlines() |
| 188 | + assert len(refs) == 1 |
| 189 | + subprocess.check_call(['ostree', 'refs', '--repo', tmpd, refs[0], '--create', buildid]) |
| 190 | + subprocess.check_call(['ostree', 'refs', '--repo', 'tmp/repo', buildid, '--delete']) |
| 191 | + subprocess.check_call(['ostree', 'pull-local', '--repo', 'tmp/repo', tmpd, buildid]) |
| 192 | + |
| 193 | + # export back all the blob refs for more efficient imports of next builds |
| 194 | + blob_refs = subprocess.check_output(['ostree', 'refs', '--repo', tmpd, |
| 195 | + '--list', 'ostree/container/blob'], |
| 196 | + encoding='utf-8').splitlines() |
| 197 | + subprocess.check_call(['ostree', 'pull-local', '--repo', 'tmp/repo', tmpd] + blob_refs) |
| 198 | + |
| 199 | + return subprocess.check_output(['ostree', 'rev-parse', '--repo', 'tmp/repo', buildid], encoding='utf-8').strip() |
| 200 | + |
| 201 | + |
| 202 | +if __name__ == '__main__': |
| 203 | + main() |
0 commit comments