budgeteer/postgres/ynab-import.go

package postgres

import (
	"context"
	"encoding/csv"
	"fmt"
	"io"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/google/uuid"
)

type YNABImport struct {
	accounts       []Account
	payees         []Payee
	categories     []GetCategoriesRow
	categoryGroups []CategoryGroup
	queries        *Queries
	budgetID       uuid.UUID
}

func NewYNABImport(context context.Context, queries *Queries, budgetID uuid.UUID) (*YNABImport, error) {
	accounts, err := queries.GetAccounts(context, budgetID)
	if err != nil {
		return nil, err
	}

	payees, err := queries.GetPayees(context, budgetID)
	if err != nil {
		return nil, err
	}

	categories, err := queries.GetCategories(context, budgetID)
	if err != nil {
		return nil, err
	}

	categoryGroups, err := queries.GetCategoryGroups(context, budgetID)
	if err != nil {
		return nil, err
	}

	return &YNABImport{
		accounts:       accounts,
		payees:         payees,
		categories:     categories,
		categoryGroups: categoryGroups,
		queries:        queries,
		budgetID:       budgetID,
	}, nil
}

// ImportAssignments expects a TSV-file as exported by YNAB in the following format:
// "Month"	"Category Group/Category"	"Category Group"	"Category"	"Budgeted"	"Activity"	"Available"
// "Apr 2019"	"Income: Next Month"	"Income"	"Next Month"	0,00€	0,00€	0,00€
//
// Activity and Available are not imported, since they are determined by the transactions and historic assignments.
func (ynab *YNABImport) ImportAssignments(context context.Context, r io.Reader) error {
	csv := csv.NewReader(r)
	csv.Comma = '\t'
	csv.LazyQuotes = true

	csvData, err := csv.ReadAll()
	if err != nil {
		return fmt.Errorf("read from tsv: %w", err)
	}

	count := 0
	for _, record := range csvData[1:] {
		dateString := record[0]
		date, err := time.Parse("Jan 2006", dateString)
		if err != nil {
			return fmt.Errorf("parse date %s: %w", dateString, err)
		}

		categoryGroup, categoryName := record[2], record[3] // also in 1 joined by :
		category, err := ynab.GetCategory(context, categoryGroup, categoryName)
		if err != nil {
			return fmt.Errorf("get category %s/%s: %w", categoryGroup, categoryName, err)
		}

		amountString := record[4]
		amount, err := GetAmount(amountString, "0,00€")
		if err != nil {
			return fmt.Errorf("parse amount %s: %w", amountString, err)
		}

		if amount.Int.Int64() == 0 {
			continue
		}

		assignment := CreateAssignmentParams{
			Date:       date,
			CategoryID: category.UUID,
			Amount:     amount,
		}
		_, err = ynab.queries.CreateAssignment(context, assignment)
		if err != nil {
			return fmt.Errorf("save assignment %v: %w", assignment, err)
		}

		count++
	}

	fmt.Printf("Imported %d assignments\n", count)

	return nil
}

type Transfer struct {
	CreateTransactionParams
	TransferToAccount *Account
	FromAccount       string
	ToAccount         string
}

// ImportTransactions expects a TSV-file as exported by YNAB in the following format:
// "Account"	"Flag"	"Date"	"Payee"	"Category Group/Category"	"Category Group"	"Category"	"Memo"	"Outflow"	"Inflow"	"Cleared"
// "Cash"	""	"11.12.2021"	"Transfer : Checking"	""	""	""	"Brought to bank"	500,00€	0,00€	"Cleared".
func (ynab *YNABImport) ImportTransactions(context context.Context, r io.Reader) error {
	csv := csv.NewReader(r)
	csv.Comma = '\t'
	csv.LazyQuotes = true

	csvData, err := csv.ReadAll()
	if err != nil {
		return fmt.Errorf("read from tsv: %w", err)
	}

	var openTransfers []Transfer

	count := 0
	for _, record := range csvData[1:] {
		transaction, err := ynab.GetTransaction(context, record)
		if err != nil {
			return err
		}

		payeeName := record[3]
		// Transaction is a transfer
		if strings.HasPrefix(payeeName, "Transfer : ") {
			err = ynab.ImportTransferTransaction(context, payeeName, transaction.CreateTransactionParams,
				&openTransfers, transaction.Account, transaction.Amount)
		} else {
			err = ynab.ImportRegularTransaction(context, payeeName, transaction.CreateTransactionParams)
		}
		if err != nil {
			return err
		}

		count++
	}

	for _, openTransfer := range openTransfers {
		fmt.Printf("Saving unmatched transfer from %s to %s on %s over %f as regular transaction\n",
			openTransfer.FromAccount, openTransfer.ToAccount, openTransfer.Date, openTransfer.Amount.GetFloat64())
		_, err = ynab.queries.CreateTransaction(context, openTransfer.CreateTransactionParams)
		if err != nil {
			return fmt.Errorf("save transaction %v: %w", openTransfer.CreateTransactionParams, err)
		}
	}

	fmt.Printf("Imported %d transactions\n", count)

	return nil
}

type NewTransaction struct {
	CreateTransactionParams
	Account *Account
}

func (ynab *YNABImport) GetTransaction(context context.Context, record []string) (NewTransaction, error) {
	accountName := record[0]
	account, err := ynab.GetAccount(context, accountName)
	if err != nil {
		return NewTransaction{}, fmt.Errorf("get account %s: %w", accountName, err)
	}

	// flag := record[1]

	dateString := record[2]
	date, err := time.Parse("02.01.2006", dateString)
	if err != nil {
		return NewTransaction{}, fmt.Errorf("parse date %s: %w", dateString, err)
	}

	categoryGroup, categoryName := record[5], record[6] // also in 4 joined by :
	category, err := ynab.GetCategory(context, categoryGroup, categoryName)
	if err != nil {
		return NewTransaction{}, fmt.Errorf("get category %s/%s: %w", categoryGroup, categoryName, err)
	}

	memo := record[7]

	outflow := record[8]
	inflow := record[9]
	amount, err := GetAmount(inflow, outflow)
	if err != nil {
		return NewTransaction{}, fmt.Errorf("parse amount from (%s/%s): %w", inflow, outflow, err)
	}

	statusEnum := TransactionStatusUncleared
	status := record[10]
	switch status {
	case "Cleared":
		statusEnum = TransactionStatusCleared
	case "Reconciled":
		statusEnum = TransactionStatusReconciled
	case "Uncleared":
	}

	return NewTransaction{
		CreateTransactionParams: CreateTransactionParams{
			Date:       date,
			Memo:       memo,
			AccountID:  account.ID,
			CategoryID: category,
			Amount:     amount,
			Status:     statusEnum,
		},
		Account: account,
	}, nil
}

func (ynab *YNABImport) ImportRegularTransaction(context context.Context, payeeName string,
	transaction CreateTransactionParams) error {
	payeeID, err := ynab.GetPayee(context, payeeName)
	if err != nil {
		return fmt.Errorf("get payee %s: %w", payeeName, err)
	}
	transaction.PayeeID = payeeID

	_, err = ynab.queries.CreateTransaction(context, transaction)
	if err != nil {
		return fmt.Errorf("save transaction %v: %w", transaction, err)
	}
	return nil
}

func (ynab *YNABImport) ImportTransferTransaction(context context.Context, payeeName string,
	transaction CreateTransactionParams, openTransfers *[]Transfer,
	account *Account, amount Numeric) error {
	transferToAccountName := payeeName[11:]
	transferToAccount, err := ynab.GetAccount(context, transferToAccountName)
	if err != nil {
		return fmt.Errorf("get transfer account %s: %w", transferToAccountName, err)
	}

	transfer := Transfer{
		transaction,
		transferToAccount,
		account.Name,
		transferToAccountName,
	}

	found := false
	for i, openTransfer := range *openTransfers {
		if openTransfer.TransferToAccount.ID != transfer.AccountID {
			continue
		}
		if openTransfer.AccountID != transfer.TransferToAccount.ID {
			continue
		}
		if openTransfer.Amount.GetFloat64() != -1*transfer.Amount.GetFloat64() {
			continue
		}

		fmt.Printf("Matched transfers from %s to %s over %f\n", account.Name, transferToAccount.Name, amount.GetFloat64())
		transfers := *openTransfers
		transfers[i] = transfers[len(transfers)-1]
		*openTransfers = transfers[:len(transfers)-1]
		found = true

		groupID := uuid.New()
		transfer.GroupID = uuid.NullUUID{UUID: groupID, Valid: true}
		openTransfer.GroupID = uuid.NullUUID{UUID: groupID, Valid: true}

		_, err = ynab.queries.CreateTransaction(context, transfer.CreateTransactionParams)
		if err != nil {
			return fmt.Errorf("save transaction %v: %w", transfer.CreateTransactionParams, err)
		}
		_, err = ynab.queries.CreateTransaction(context, openTransfer.CreateTransactionParams)
		if err != nil {
			return fmt.Errorf("save transaction %v: %w", openTransfer.CreateTransactionParams, err)
		}
		break
	}

	if !found {
		*openTransfers = append(*openTransfers, transfer)
	}
	return nil
}

func trimLastChar(s string) string {
	r, size := utf8.DecodeLastRuneInString(s)
	if r == utf8.RuneError && (size == 0 || size == 1) {
		size = 0
	}
	return s[:len(s)-size]
}

func ParseNumeric(text string) (Numeric, error) {
	// Remove trailing currency
	text = trimLastChar(text)

	// Unify decimal separator
	text = strings.Replace(text, ",", ".", 1)

	num := Numeric{}
	err := num.Set(text)
	if err != nil {
		return num, fmt.Errorf("parse numeric %s: %w", text, err)
	}

	return num, nil
}

func GetAmount(inflow string, outflow string) (Numeric, error) {
	in, err := ParseNumeric(inflow)
	if err != nil {
		return in, err
	}

	if !in.IsZero() {
		return in, nil
	}

	// if inflow is zero, use outflow
	out, err := ParseNumeric("-" + outflow)
	if err != nil {
		return out, err
	}
	return out, nil
}

func (ynab *YNABImport) GetAccount(context context.Context, name string) (*Account, error) {
	for _, acc := range ynab.accounts {
		if acc.Name == name {
			return &acc, nil
		}
	}

	account, err := ynab.queries.CreateAccount(context, CreateAccountParams{Name: name, BudgetID: ynab.budgetID})
	if err != nil {
		return nil, err
	}

	ynab.accounts = append(ynab.accounts, account)
	return &account, nil
}

func (ynab *YNABImport) GetPayee(context context.Context, name string) (uuid.NullUUID, error) {
	if name == "" {
		return uuid.NullUUID{}, nil
	}

	for _, pay := range ynab.payees {
		if pay.Name == name {
			return uuid.NullUUID{UUID: pay.ID, Valid: true}, nil
		}
	}

	payee, err := ynab.queries.CreatePayee(context, CreatePayeeParams{Name: name, BudgetID: ynab.budgetID})
	if err != nil {
		return uuid.NullUUID{}, err
	}

	ynab.payees = append(ynab.payees, payee)
	return uuid.NullUUID{UUID: payee.ID, Valid: true}, nil
}

func (ynab *YNABImport) GetCategory(context context.Context, group string, name string) (uuid.NullUUID, error) { //nolint
	if group == "" || name == "" {
		return uuid.NullUUID{}, nil
	}

	for _, category := range ynab.categories {
		if category.Name == name && category.Group == group {
			return uuid.NullUUID{UUID: category.ID, Valid: true}, nil
		}
	}

	var categoryGroup CategoryGroup
	for _, existingGroup := range ynab.categoryGroups {
		if existingGroup.Name == group {
			categoryGroup = existingGroup
		}
	}

	if categoryGroup.Name == "" {
		newGroup := CreateCategoryGroupParams{Name: group, BudgetID: ynab.budgetID}
		var err error
		categoryGroup, err = ynab.queries.CreateCategoryGroup(context, newGroup)
		if err != nil {
			return uuid.NullUUID{}, err
		}
		ynab.categoryGroups = append(ynab.categoryGroups, categoryGroup)
	}

	newCategory := CreateCategoryParams{Name: name, CategoryGroupID: categoryGroup.ID}
	category, err := ynab.queries.CreateCategory(context, newCategory)
	if err != nil {
		return uuid.NullUUID{}, err
	}

	getCategory := GetCategoriesRow{
		ID:              category.ID,
		CategoryGroupID: category.CategoryGroupID,
		Name:            category.Name,
		Group:           categoryGroup.Name,
	}
	ynab.categories = append(ynab.categories, getCategory)
	return uuid.NullUUID{UUID: category.ID, Valid: true}, nil
}