commit

Author: bufdev

README.md

@@ -186,14 +186,12 @@ To keep data current, the Flex Query API provides the latest 365 days. To add ol
 
 ## Data Storage
 
-All data is cached as protobuf-JSON files under the data directory (`<data_dir>/v1/` as configured in `ibctl.yaml`). Each file stores newline-separated proto JSON (one message per line), serialized using `protojson` with proto field names. `metadata.json` is a single message.
+Raw API data is cached as protobuf-JSON files under the data directory (`<data_dir>/v1/` as configured in `ibctl.yaml`). Each file stores newline-separated proto JSON (one message per line), serialized using `protojson` with proto field names. Tax lots and derived computations are performed at read time from the merged data (Activity Statement CSVs + cached API data).
 
 | File | Protobuf Message | Description |
 |------|-----------------|-------------|
 | `trades.json` | `ibctl.data.v1.Trade` | All trades from the IBKR Flex Query. Each trade includes trade ID, dates, symbol, side (buy/sell), quantity, price, proceeds, commission, currency code, and FIFO realized P&L. |
 | `positions.json` | `ibctl.data.v1.Position` | Open positions as reported by IBKR, including quantity, cost basis price, market price, market value, currency code, and unrealized P&L. |
-| `tax_lots.json` | `ibctl.data.v1.TaxLot` | FIFO tax lots computed from trades. Each lot tracks symbol, open date, remaining quantity, cost basis price, and currency code. Long-term status (held >= 1 year) is computed dynamically at display time. |
 | `exchange_rates.json` | `ibctl.data.v1.ExchangeRate` | Currency exchange rates with date, base/quote currency codes, rate (units + micros), and provider (ibkr or [frankfurter.dev](https://frankfurter.dev)). |
-| `metadata.json` | `ibctl.data.v1.Metadata` | Download timestamp, whether computed positions matched IBKR-reported positions, and any verification discrepancy notes. |
 
-Monetary values use `standard.money.v1.Money` with units and micros (6 decimal places). Dates use `standard.time.v1.Date` with year, month, and day fields. Timestamps use `google.protobuf.Timestamp`.
+Monetary values use `standard.money.v1.Money` with units and micros (6 decimal places). Dates use `standard.time.v1.Date` with year, month, and day fields.

cmd/ibctl/internal/command/holdings/holdingsoverview/holdingsoverview.go

@@ -15,8 +15,10 @@ import (
 	"github.com/bufdev/ibctl/internal/ibctl/ibctlconfig"
 	"github.com/bufdev/ibctl/internal/ibctl/ibctlholdings"
 	"github.com/bufdev/ibctl/internal/ibctl/ibctlmerge"
+	"github.com/bufdev/ibctl/internal/ibctl/ibctltaxlot"
 	"github.com/bufdev/ibctl/internal/pkg/cliio"
 	"github.com/bufdev/ibctl/internal/pkg/ibkractivitycsv"
+	"github.com/bufdev/ibctl/internal/pkg/mathpb"
 	"github.com/spf13/pflag"
 )
 
@@ -86,31 +88,71 @@ func run(ctx context.Context, container appext.Container, flags *flags) error {
 		return err
 	}
 	// Compute the holdings overview from merged data.
-	holdingsOverview, err := ibctlholdings.GetHoldingsOverview(mergedData.Trades, mergedData.Positions, config)
+	result, err := ibctlholdings.GetHoldingsOverview(mergedData.Trades, mergedData.Positions, config)
 	if err != nil {
 		return err
 	}
+	// Log any data inconsistencies detected during computation.
+	logger := container.Logger()
+	for _, unmatched := range result.UnmatchedSells {
+		logger.Warn("unmatched sell (buy likely before data window)",
+			"symbol", unmatched.Symbol,
+			"unmatched_quantity", mathpb.ToString(unmatched.UnmatchedQuantity),
+		)
+	}
+	for _, discrepancy := range result.PositionDiscrepancies {
+		logPositionDiscrepancy(container, discrepancy)
+	}
 	// Write output in the requested format.
 	writer := os.Stdout
 	switch format {
 	case cliio.FormatTable:
 		headers := ibctlholdings.HoldingsOverviewHeaders()
-		rows := make([][]string, 0, len(holdingsOverview))
-		for _, h := range holdingsOverview {
+		rows := make([][]string, 0, len(result.Holdings))
+		for _, h := range result.Holdings {
 			rows = append(rows, ibctlholdings.HoldingOverviewToRow(h))
 		}
 		return cliio.WriteTable(writer, headers, rows)
 	case cliio.FormatCSV:
 		headers := ibctlholdings.HoldingsOverviewHeaders()
-		records := make([][]string, 0, len(holdingsOverview)+1)
+		records := make([][]string, 0, len(result.Holdings)+1)
 		records = append(records, headers)
-		for _, h := range holdingsOverview {
+		for _, h := range result.Holdings {
 			records = append(records, ibctlholdings.HoldingOverviewToRow(h))
 		}
 		return cliio.WriteCSVRecords(writer, records)
 	case cliio.FormatJSON:
-		return cliio.WriteJSON(writer, holdingsOverview...)
+		return cliio.WriteJSON(writer, result.Holdings...)
 	default:
 		return appcmd.NewInvalidArgumentErrorf("unsupported format: %s", format)
 	}
 }
+
+// logPositionDiscrepancy logs a structured position discrepancy as a warning.
+func logPositionDiscrepancy(container appext.Container, d ibctltaxlot.PositionDiscrepancy) {
+	logger := container.Logger()
+	switch d.Type {
+	case ibctltaxlot.DiscrepancyTypeQuantity:
+		logger.Warn("position quantity mismatch",
+			"symbol", d.Symbol,
+			"computed", d.ComputedValue,
+			"reported", d.ReportedValue,
+		)
+	case ibctltaxlot.DiscrepancyTypeCostBasis:
+		logger.Warn("position cost basis mismatch",
+			"symbol", d.Symbol,
+			"computed", d.ComputedValue,
+			"reported", d.ReportedValue,
+		)
+	case ibctltaxlot.DiscrepancyTypeComputedOnly:
+		logger.Warn("position computed but not reported by IBKR",
+			"symbol", d.Symbol,
+			"computed_quantity", d.ComputedValue,
+		)
+	case ibctltaxlot.DiscrepancyTypeReportedOnly:
+		logger.Warn("position reported by IBKR but not in computed data",
+			"symbol", d.Symbol,
+			"reported_quantity", d.ReportedValue,
+		)
+	}
+}

internal/ibctl/ibctldownload/ibctldownload.go

@@ -21,15 +21,13 @@ import (
 
 	datav1 "github.com/bufdev/ibctl/internal/gen/proto/go/ibctl/data/v1"
 	"github.com/bufdev/ibctl/internal/ibctl/ibctlconfig"
-	"github.com/bufdev/ibctl/internal/ibctl/ibctltaxlot"
 	"github.com/bufdev/ibctl/internal/pkg/frankfurter"
 	"github.com/bufdev/ibctl/internal/pkg/ibkrflexquery"
 	"github.com/bufdev/ibctl/internal/pkg/mathpb"
 	"github.com/bufdev/ibctl/internal/pkg/moneypb"
 	"github.com/bufdev/ibctl/internal/pkg/protoio"
 	"github.com/bufdev/ibctl/internal/pkg/timepb"
 	"github.com/bufdev/ibctl/internal/standard/xtime"
-	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // Downloader is the interface for downloading and caching IBKR data.
@@ -105,11 +103,12 @@ func (d *downloader) Download(ctx context.Context) error {
 }
 
 // processAndWrite converts XML data to protos, merges with existing cached data,
-// computes tax lots, verifies positions, and writes all JSON data files.
+// and writes the raw data files (trades, positions, exchange rates).
 // This is idempotent — running it multiple times with overlapping data produces
 // the same result. Trades are deduplicated by trade ID, exchange rates by
 // date+currency pair. Positions are always overwritten (they are a point-in-time
-// snapshot). Tax lots and metadata are recomputed from the full merged trade set.
+// snapshot). Tax lots and derived computations happen at read time in commands
+// that have access to the full merged data (including Activity Statement CSVs).
 func (d *downloader) processAndWrite(
 	ctx context.Context,
 	xmlTrades []ibkrflexquery.XMLTrade,
@@ -152,41 +151,7 @@ func (d *downloader) processAndWrite(
 		return fmt.Errorf("writing exchange rates: %w", err)
 	}
 	d.logger.Info("exchange rates written", "count", len(exchangeRates), "path", exchangeRatesPath)
-
-	// Compute tax lots from the full merged trade set.
-	taxLots, err := ibctltaxlot.ComputeTaxLots(trades)
-	if err != nil {
-		return fmt.Errorf("computing tax lots: %w", err)
-	}
-	taxLotsPath := filepath.Join(d.dataDirV1Path, "tax_lots.json")
-	if err := protoio.WriteMessagesJSON(taxLotsPath, taxLots); err != nil {
-		return fmt.Errorf("writing tax lots: %w", err)
-	}
-	d.logger.Info("tax lots written", "count", len(taxLots), "path", taxLotsPath)
-
-	// Compute positions from tax lots and verify against IBKR-reported positions.
-	computedPositions := ibctltaxlot.ComputePositions(taxLots)
-	verificationNotes := ibctltaxlot.VerifyPositions(computedPositions, positions)
-	positionsVerified := len(verificationNotes) == 0
-	if !positionsVerified {
-		for _, note := range verificationNotes {
-			d.logger.Warn("position verification", "note", note)
-		}
-	} else {
-		d.logger.Info("all positions verified successfully")
-	}
-
-	// Write metadata.json.
-	metadata := &datav1.Metadata{
-		DownloadTime:      timestamppb.Now(),
-		PositionsVerified: positionsVerified,
-		VerificationNotes: verificationNotes,
-	}
-	metadataPath := filepath.Join(d.dataDirV1Path, "metadata.json")
-	if err := protoio.WriteMessageJSON(metadataPath, metadata); err != nil {
-		return fmt.Errorf("writing metadata: %w", err)
-	}
-	d.logger.Info("download complete", "positions_verified", positionsVerified)
+	d.logger.Info("download complete")
 	return nil
 }
 

internal/ibctl/ibctlholdings/ibctlholdings.go

@@ -16,6 +16,17 @@ import (
 	"github.com/bufdev/ibctl/internal/pkg/moneypb"
 )
 
+// HoldingsResult contains the holdings overview along with any data
+// inconsistencies detected during computation.
+type HoldingsResult struct {
+	// Holdings is the list of holdings for display.
+	Holdings []*HoldingOverview
+	// UnmatchedSells records sells that could not be matched to buy lots.
+	UnmatchedSells []ibctltaxlot.UnmatchedSell
+	// PositionDiscrepancies records mismatches between computed and IBKR-reported positions.
+	PositionDiscrepancies []ibctltaxlot.PositionDiscrepancy
+}
+
 // HoldingOverview represents a single holding for display.
 type HoldingOverview struct {
 	// Symbol is the ticker symbol.
@@ -54,15 +65,17 @@ func HoldingOverviewToRow(h *HoldingOverview) []string {
 
 // GetHoldingsOverview computes the holdings overview from merged trade and position data.
 // Trades are used to compute FIFO tax lots and average cost basis.
-// Positions provide the latest market prices.
-func GetHoldingsOverview(trades []*datav1.Trade, positions []*datav1.Position, config *ibctlconfig.Config) ([]*HoldingOverview, error) {
+// Positions provide the latest market prices and are used for verification.
+func GetHoldingsOverview(trades []*datav1.Trade, positions []*datav1.Position, config *ibctlconfig.Config) (*HoldingsResult, error) {
 	// Compute tax lots from merged trades.
-	taxLots, err := ibctltaxlot.ComputeTaxLots(trades)
+	taxLotResult, err := ibctltaxlot.ComputeTaxLots(trades)
 	if err != nil {
 		return nil, err
 	}
 	// Compute positions from tax lots (for average cost basis).
-	computedPositions := ibctltaxlot.ComputePositions(taxLots)
+	computedPositions := ibctltaxlot.ComputePositions(taxLotResult.TaxLots)
+	// Verify computed positions against IBKR-reported positions.
+	discrepancies := ibctltaxlot.VerifyPositions(computedPositions, positions)
 
 	// Build a map of market prices from IBKR-reported positions.
 	marketPrices := make(map[string]string, len(positions))
@@ -93,5 +106,9 @@ func GetHoldingsOverview(trades []*datav1.Trade, positions []*datav1.Position, c
 	sort.Slice(holdings, func(i, j int) bool {
 		return holdings[i].Symbol < holdings[j].Symbol
 	})
-	return holdings, nil
+	return &HoldingsResult{
+		Holdings:              holdings,
+		UnmatchedSells:        taxLotResult.UnmatchedSells,
+		PositionDiscrepancies: discrepancies,
+	}, nil
 }