[WIP] add support for copying recrods from local git cach to calypr remote …

cmd/copyrecords/api.go

@@ -77,7 +77,9 @@ func (r *rawIndexAPI) List(ctx context.Context, opts syservices.ListRecordsOptio
 	if opts.Limit != 0 {
 		params.Set("limit", fmt.Sprintf("%d", opts.Limit))
 	}
-	if opts.Page != 0 {
+	if opts.Start != "" {
+		params.Set("start", opts.Start)
+	} else if opts.Page != 0 {
 		params.Set("page", fmt.Sprintf("%d", opts.Page))
 	}
 	var out copyListRecordsResponse

cmd/copyrecords/local_target.go

@@ -0,0 +1,137 @@
+package copyrecords
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	drsapi "github.com/calypr/syfon/apigen/client/drs"
+	syservices "github.com/calypr/syfon/client/services"
+)
+
+type localIndexAPI struct{}
+
+func (localIndexAPI) List(ctx context.Context, opts syservices.ListRecordsOptions) (copyListRecordsResponse, error) {
+	return copyListRecordsResponse{}, fmt.Errorf("local target does not support source listing")
+}
+
+func (localIndexAPI) BulkDocuments(ctx context.Context, dids []string) ([]copyRecord, error) {
+	return nil, nil
+}
+
+func (localIndexAPI) BulkHashes(ctx context.Context, hashes []string) (copyBulkHashesResponse, error) {
+	results := make(map[string][]copyRecord, len(hashes))
+	for _, raw := range hashes {
+		query := strings.TrimSpace(raw)
+		oid := strings.TrimPrefix(query, "sha256:")
+		if oid == "" {
+			continue
+		}
+		obj, err := readLocalDRSObject(oid)
+		if err != nil {
+			continue
+		}
+		results[query] = []copyRecord{copyRecordFromLocalObject(obj)}
+	}
+	return copyBulkHashesResponse{Results: results}, nil
+}
+
+func (localIndexAPI) CreateBulk(ctx context.Context, req copyBulkCreateRequest) (copyListRecordsResponse, error) {
+	written := make([]copyRecord, 0, len(req.Records))
+	skippedMissingHash := 0
+	for _, rec := range req.Records {
+		oid := localObjectKeyForCopyRecord(rec)
+		if oid == "" {
+			skippedMissingHash++
+			continue
+		}
+		obj := localObjectFromCopyRecord(rec)
+		if err := writeLocalDRSObject(oid, obj); err != nil {
+			return copyListRecordsResponse{}, fmt.Errorf("write local DRS object for oid %s: %w", oid, err)
+		}
+		written = append(written, rec)
+	}
+	if skippedMissingHash > 0 {
+		fmt.Fprintf(os.Stderr, "copy-records: skipped %d source records that cannot be written locally because they have no valid sha256 hash\n", skippedMissingHash)
+	}
+	return copyListRecordsResponse{Records: &written}, nil
+}
+
+func localObjectKeyForCopyRecord(rec copyRecord) string {
+	if sha := copyRecordSHA256(rec); isHexSHA256(sha) {
+		return sha
+	}
+	return ""
+}
+
+func isHexSHA256(raw string) bool {
+	if len(raw) != 64 {
+		return false
+	}
+	for _, r := range raw {
+		switch {
+		case r >= '0' && r <= '9':
+		case r >= 'a' && r <= 'f':
+		case r >= 'A' && r <= 'F':
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+func localObjectFromCopyRecord(rec copyRecord) *drsapi.DrsObject {
+	checksums := make([]drsapi.Checksum, 0)
+	if rec.Hashes != nil {
+		for typ, sum := range *rec.Hashes {
+			typ = strings.TrimSpace(typ)
+			sum = strings.TrimSpace(sum)
+			if typ == "" || sum == "" {
+				continue
+			}
+			checksums = append(checksums, drsapi.Checksum{Type: typ, Checksum: strings.TrimPrefix(sum, typ+":")})
+		}
+	}
+	if len(checksums) == 0 {
+		if sha := copyRecordSHA256(rec); sha != "" {
+			checksums = append(checksums, drsapi.Checksum{Type: "sha256", Checksum: sha})
+		}
+	}
+
+	obj := &drsapi.DrsObject{
+		AccessMethods:    rec.AccessMethods,
+		Checksums:        checksums,
+		ControlledAccess: rec.ControlledAccess,
+		Description:      rec.Description,
+		Id:               strings.TrimSpace(rec.Did),
+		Name:             rec.Name,
+		SelfUri:          "drs://" + strings.TrimSpace(rec.Did),
+		Version:          rec.Version,
+	}
+	if rec.Size != nil {
+		obj.Size = *rec.Size
+	}
+	if created := parseCopyRecordTime(rec.CreatedTime); created != nil {
+		obj.CreatedTime = *created
+	}
+	obj.UpdatedTime = parseCopyRecordTime(rec.UpdatedTime)
+	return obj
+}
+
+func parseCopyRecordTime(raw *string) *time.Time {
+	if raw == nil {
+		return nil
+	}
+	value := strings.TrimSpace(*raw)
+	if value == "" {
+		return nil
+	}
+	for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05 -0700 MST"} {
+		if parsed, err := time.Parse(layout, value); err == nil {
+			return &parsed
+		}
+	}
+	return nil
+}

cmd/copyrecords/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"sort"
 	"strings"
 
 	"github.com/calypr/git-drs/internal/config"
@@ -29,12 +30,15 @@ var (
 	loadLocalSource = func(ctx context.Context, org, project string) ([]copyRecord, error) {
 		return loadLocalSourceRecords(org, project)
 	}
+	newLocalTargetAPI = func() indexAPI {
+		return localIndexAPI{}
+	}
 )
 
 var Cmd = &cobra.Command{
 	Use:   "copy-records [source-remote] <target-remote> <organization/project>",
 	Short: "Copy Syfon records between remotes for one organization/project scope",
-	Long:  "Read source records from either a source remote or the current repo's local DRS metadata and bulk load them into a target Syfon instance, only merging controlled_access and access_methods for records that already exist on the target. Use `git drs copy-records local <target-remote> <organization/project>` to copy local repo records.",
+	Long:  "Read source records from either a source remote or the current repo's local DRS metadata and bulk load them into a target Syfon instance or the current repo's local DRS metadata, only merging controlled_access and access_methods for records that already exist on the target. Use `git drs copy-records local <target-remote> <organization/project>` to copy local repo records, or `git drs copy-records <source-remote> local <organization/project>` to copy remote records into local repo metadata.",
 	Args:  cobra.RangeArgs(2, 3),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		logger := drslog.GetLogger()
@@ -58,23 +62,33 @@ var Cmd = &cobra.Command{
 		if strings.TrimSpace(targetRemote) == "" {
 			return fmt.Errorf("target remote is required")
 		}
+		targetIsLocal := isLocalSentinel(targetRemote)
 		dstRemoteName := config.Remote(strings.TrimSpace(targetRemote))
 
 		org, proj, err := parseScopeArg(scopeArg)
 		if err != nil {
 			return err
 		}
 
-		dstCtx, err := newCopyRuntime(cfg, dstRemoteName, logger)
-		if err != nil {
-			return fmt.Errorf("error creating target client: %w", err)
+		var dstAPI indexAPI
+		if targetIsLocal {
+			dstAPI = newLocalTargetAPI()
+		} else {
+			dstCtx, err := newCopyRuntime(cfg, dstRemoteName, logger)
+			if err != nil {
+				return fmt.Errorf("error creating target client: %w", err)
+			}
+			dstAPI = newCopyIndexAPI(dstCtx.Client.Requestor())
 		}
 
 		var (
 			sourceRecords []copyRecord
 			sourceLabel   string
 		)
-		if strings.EqualFold(strings.TrimSpace(sourceRemote), "local") {
+		if isLocalSentinel(sourceRemote) {
+			if targetIsLocal {
+				return fmt.Errorf("source and target cannot both be local")
+			}
 			sourceRecords, err = loadLocalSource(cmd.Context(), org, proj)
 			if err != nil {
 				return err
@@ -85,29 +99,50 @@ var Cmd = &cobra.Command{
 			if err != nil {
 				return fmt.Errorf("error resolving source remote: %w", err)
 			}
-			if srcRemoteName == dstRemoteName {
+			if !targetIsLocal && srcRemoteName == dstRemoteName {
 				return fmt.Errorf("source and target remotes must be different")
 			}
 			srcCfg := cfg.GetRemote(srcRemoteName)
 			if srcCfg == nil {
-				return fmt.Errorf("source remote %q not found", srcRemoteName)
+				return fmt.Errorf("source remote %q not found. Available remotes: %s. Run `git drs remote list` in this repository to inspect configured git-drs remotes", srcRemoteName, configuredRemoteList(cfg))
 			}
 			srcCtx, err := newCopyRuntime(cfg, srcRemoteName, logger)
 			if err != nil {
 				return fmt.Errorf("error creating source client: %w", err)
 			}
-			sourceRecords, err = listSourceRecordsByControlledAccess(cmd.Context(), newCopyIndexAPI(srcCtx.Client.Requestor()), org, proj, batchSize)
+			stats, err := copyProjectRecordsFromSourceIndex(
+				cmd.Context(),
+				logger,
+				newCopyIndexAPI(srcCtx.Client.Requestor()),
+				dstAPI,
+				org,
+				proj,
+				batchSize,
+				overwriteName,
+			)
 			if err != nil {
 				return err
 			}
 			sourceLabel = string(srcRemoteName)
+			logger.Info("copy-records complete",
+				"source_remote", sourceLabel,
+				"target_remote", dstRemoteName,
+				"organization", org,
+				"project", proj,
+				"source_seen", stats.SourceSeen,
+				"created", stats.Created,
+				"updated", stats.Updated,
+				"unchanged", stats.Unchanged,
+				"written", stats.Written,
+			)
+			return nil
 		}
 
 		stats, err := copyProjectRecords(
 			cmd.Context(),
 			logger,
 			sourceRecords,
-			newCopyIndexAPI(dstCtx.Client.Requestor()),
+			dstAPI,
 			org,
 			proj,
 			batchSize,
@@ -116,7 +151,6 @@ var Cmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-
 		logger.Info("copy-records complete",
 			"source_remote", sourceLabel,
 			"target_remote", dstRemoteName,
@@ -133,6 +167,22 @@ var Cmd = &cobra.Command{
 }
 
 func init() {
-	Cmd.Flags().IntVar(&batchSize, "batch-size", 250, "records per source page and target bulk write")
+	Cmd.Flags().IntVar(&batchSize, "batch-size", defaultCopyBatchSize, "records per source page and target bulk write")
 	Cmd.Flags().BoolVar(&overwriteName, "overwrite-name", false, "for existing target records, replace target name with the source value")
 }
+
+func isLocalSentinel(remote string) bool {
+	return strings.EqualFold(strings.TrimSpace(remote), "local")
+}
+
+func configuredRemoteList(cfg *config.Config) string {
+	if cfg == nil || len(cfg.Remotes) == 0 {
+		return "<none>"
+	}
+	names := make([]string, 0, len(cfg.Remotes))
+	for name := range cfg.Remotes {
+		names = append(names, string(name))
+	}
+	sort.Strings(names)
+	return strings.Join(names, ", ")
+}