Merge pull request 'Import transfers as actual Transfers' (#1) from handle-transfers into master

Reviewed-on: #1

.drone.yml

@@ -22,6 +22,11 @@ steps:
     dockerfile: build/Dockerfile
     tags: 
       - latest
+   when:
+    event:
+      exclude:
+      - pull_request
+
 
 image_pull_secrets:
 - hub.javil.eu 

postgres/models.go

@@ -64,6 +64,7 @@ type Transaction struct {
 	AccountID  uuid.UUID
 	CategoryID uuid.NullUUID
 	PayeeID    uuid.NullUUID
+	GroupID    uuid.NullUUID
 }
 
 type TransactionsByMonth struct {

postgres/queries/transactions.sql

@@ -4,8 +4,8 @@ WHERE id = $1;
 
 -- name: CreateTransaction :one
 INSERT INTO transactions
-(date, memo, amount, account_id, payee_id, category_id)
+(date, memo, amount, account_id, payee_id, category_id, group_id)
-VALUES ($1, $2, $3, $4, $5, $6)
+VALUES ($1, $2, $3, $4, $5, $6, $7)
 RETURNING *;
 
 -- name: UpdateTransaction :exec
@@ -23,7 +23,7 @@ DELETE FROM transactions
 WHERE id = $1;
 
 -- name: GetTransactionsForBudget :many
-SELECT  transactions.id, transactions.date, transactions.memo, transactions.amount,
+SELECT  transactions.id, transactions.date, transactions.memo, transactions.amount, transactions.group_id,
         accounts.name as account, COALESCE(payees.name, '') as payee, COALESCE(category_groups.name, '') as category_group, COALESCE(categories.name, '') as category
 FROM transactions 
 INNER JOIN accounts ON accounts.id = transactions.account_id
@@ -35,7 +35,7 @@ ORDER BY transactions.date DESC
 LIMIT 200;
 
 -- name: GetTransactionsForAccount :many
-SELECT  transactions.id, transactions.date, transactions.memo, transactions.amount,
+SELECT  transactions.id, transactions.date, transactions.memo, transactions.amount, transactions.group_id,
         accounts.name as account, COALESCE(payees.name, '') as payee, COALESCE(category_groups.name, '') as category_group, COALESCE(categories.name, '') as category
 FROM transactions 
 INNER JOIN accounts ON accounts.id = transactions.account_id

postgres/schema/0012_add-group-id.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+ALTER TABLE transactions ADD COLUMN group_id uuid NULL;
+
+-- +goose Down
+ALTER TABLE transactions DROP COLUMN group_id;

postgres/transactions.sql.go

@@ -12,9 +12,9 @@ import (
 
 const createTransaction = `-- name: CreateTransaction :one
 INSERT INTO transactions
-(date, memo, amount, account_id, payee_id, category_id)
+(date, memo, amount, account_id, payee_id, category_id, group_id)
-VALUES ($1, $2, $3, $4, $5, $6)
+VALUES ($1, $2, $3, $4, $5, $6, $7)
-RETURNING id, date, memo, amount, account_id, category_id, payee_id
+RETURNING id, date, memo, amount, account_id, category_id, payee_id, group_id
 `
 
 type CreateTransactionParams struct {
@@ -24,6 +24,7 @@ type CreateTransactionParams struct {
 	AccountID  uuid.UUID
 	PayeeID    uuid.NullUUID
 	CategoryID uuid.NullUUID
+	GroupID    uuid.NullUUID
 }
 
 func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (Transaction, error) {
@@ -34,6 +35,7 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa
 		arg.AccountID,
 		arg.PayeeID,
 		arg.CategoryID,
+		arg.GroupID,
 	)
 	var i Transaction
 	err := row.Scan(
@@ -44,6 +46,7 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa
 		&i.AccountID,
 		&i.CategoryID,
 		&i.PayeeID,
+		&i.GroupID,
 	)
 	return i, err
 }
@@ -74,7 +77,7 @@ func (q *Queries) DeleteTransaction(ctx context.Context, id uuid.UUID) error {
 }
 
 const getTransaction = `-- name: GetTransaction :one
-SELECT id, date, memo, amount, account_id, category_id, payee_id FROM transactions
+SELECT id, date, memo, amount, account_id, category_id, payee_id, group_id FROM transactions
 WHERE id = $1
 `
 
@@ -89,6 +92,7 @@ func (q *Queries) GetTransaction(ctx context.Context, id uuid.UUID) (Transaction
 		&i.AccountID,
 		&i.CategoryID,
 		&i.PayeeID,
+		&i.GroupID,
 	)
 	return i, err
 }
@@ -128,7 +132,7 @@ func (q *Queries) GetTransactionsByMonthAndCategory(ctx context.Context, budgetI
 }
 
 const getTransactionsForAccount = `-- name: GetTransactionsForAccount :many
-SELECT  transactions.id, transactions.date, transactions.memo, transactions.amount,
+SELECT  transactions.id, transactions.date, transactions.memo, transactions.amount, transactions.group_id,
         accounts.name as account, COALESCE(payees.name, '') as payee, COALESCE(category_groups.name, '') as category_group, COALESCE(categories.name, '') as category
 FROM transactions 
 INNER JOIN accounts ON accounts.id = transactions.account_id
@@ -145,6 +149,7 @@ type GetTransactionsForAccountRow struct {
 	Date          time.Time
 	Memo          string
 	Amount        Numeric
+	GroupID       uuid.NullUUID
 	Account       string
 	Payee         string
 	CategoryGroup string
@@ -165,6 +170,7 @@ func (q *Queries) GetTransactionsForAccount(ctx context.Context, accountID uuid.
 			&i.Date,
 			&i.Memo,
 			&i.Amount,
+			&i.GroupID,
 			&i.Account,
 			&i.Payee,
 			&i.CategoryGroup,
@@ -184,7 +190,7 @@ func (q *Queries) GetTransactionsForAccount(ctx context.Context, accountID uuid.
 }
 
 const getTransactionsForBudget = `-- name: GetTransactionsForBudget :many
-SELECT  transactions.id, transactions.date, transactions.memo, transactions.amount,
+SELECT  transactions.id, transactions.date, transactions.memo, transactions.amount, transactions.group_id,
         accounts.name as account, COALESCE(payees.name, '') as payee, COALESCE(category_groups.name, '') as category_group, COALESCE(categories.name, '') as category
 FROM transactions 
 INNER JOIN accounts ON accounts.id = transactions.account_id
@@ -201,6 +207,7 @@ type GetTransactionsForBudgetRow struct {
 	Date          time.Time
 	Memo          string
 	Amount        Numeric
+	GroupID       uuid.NullUUID
 	Account       string
 	Payee         string
 	CategoryGroup string
@@ -221,6 +228,7 @@ func (q *Queries) GetTransactionsForBudget(ctx context.Context, budgetID uuid.UU
 			&i.Date,
 			&i.Memo,
 			&i.Amount,
+			&i.GroupID,
 			&i.Account,
 			&i.Payee,
 			&i.CategoryGroup,

postgres/ynab-import.go

@@ -113,6 +113,13 @@ func (ynab *YNABImport) ImportAssignments(r io.Reader) error {
 	return nil
 }
 
+type Transfer struct {
+	CreateTransactionParams
+	TransferToAccount *Account
+	FromAccount       string
+	ToAccount         string
+}
+
 // ImportTransactions expects a TSV-file as exported by YNAB in the following format:
 
 func (ynab *YNABImport) ImportTransactions(r io.Reader) error {
@@ -125,6 +132,8 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error {
 		return fmt.Errorf("could not read from tsv: %w", err)
 	}
 
+	var openTransfers []Transfer
+
 	count := 0
 	for _, record := range csvData[1:] {
 		accountName := record[0]
@@ -141,12 +150,6 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error {
 			return fmt.Errorf("could not parse date %s: %w", dateString, err)
 		}
 
-		payeeName := record[3]
-		payeeID, err := ynab.GetPayee(payeeName)
-		if err != nil {
-			return fmt.Errorf("could not get payee %s: %w", payeeName, err)
-		}
-
 		categoryGroup, categoryName := record[5], record[6] //also in 4 joined by :
 		category, err := ynab.GetCategory(categoryGroup, categoryName)
 		if err != nil {
@@ -162,24 +165,91 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error {
 			return fmt.Errorf("could not parse amount from (%s/%s): %w", inflow, outflow, err)
 		}
 
-		//status := record[10]
-
 		transaction := CreateTransactionParams{
 			Date:       date,
 			Memo:       memo,
 			AccountID:  account.ID,
-			PayeeID:    payeeID,
 			CategoryID: category,
 			Amount:     amount,
 		}
+
+		payeeName := record[3]
+		if strings.HasPrefix(payeeName, "Transfer : ") {
+			// Transaction is a transfer to
+			transferToAccountName := payeeName[11:]
+			transferToAccount, err := ynab.GetAccount(transferToAccountName)
+			if err != nil {
+				return fmt.Errorf("Could not get transfer account %s: %w", transferToAccountName, err)
+			}
+
+			transfer := Transfer{
+				transaction,
+				transferToAccount,
+				accountName,
+				transferToAccountName,
+			}
+
+			found := false
+			for i, openTransfer := range openTransfers {
+				if openTransfer.TransferToAccount.ID != transfer.AccountID {
+					continue
+				}
+				if openTransfer.AccountID != transfer.TransferToAccount.ID {
+					continue
+				}
+				if openTransfer.Amount.GetFloat64() != -1*transfer.Amount.GetFloat64() {
+					continue
+				}
+
+				fmt.Printf("Matched transfers from %s to %s over %f\n", account.Name, transferToAccount.Name, amount.GetFloat64())
+				openTransfers[i] = openTransfers[len(openTransfers)-1]
+				openTransfers = openTransfers[:len(openTransfers)-1]
+				found = true
+
+				groupID := uuid.New()
+				transfer.GroupID = uuid.NullUUID{UUID: groupID, Valid: true}
+				openTransfer.GroupID = uuid.NullUUID{UUID: groupID, Valid: true}
+
+				_, err = ynab.queries.CreateTransaction(ynab.Context, transfer.CreateTransactionParams)
+				if err != nil {
+					return fmt.Errorf("could not save transaction %v: %w", transfer.CreateTransactionParams, err)
+				}
+				_, err = ynab.queries.CreateTransaction(ynab.Context, openTransfer.CreateTransactionParams)
+				if err != nil {
+					return fmt.Errorf("could not save transaction %v: %w", openTransfer.CreateTransactionParams, err)
+				}
+				break
+			}
+
+			if !found {
+				openTransfers = append(openTransfers, transfer)
+			}
+		} else {
+			payeeID, err := ynab.GetPayee(payeeName)
+			if err != nil {
+				return fmt.Errorf("could not get payee %s: %w", payeeName, err)
+			}
+			transaction.PayeeID = payeeID
+
 			_, err = ynab.queries.CreateTransaction(ynab.Context, transaction)
 			if err != nil {
 				return fmt.Errorf("could not save transaction %v: %w", transaction, err)
 			}
+		}
+
+		//status := record[10]
 
 		count++
 	}
 
+	for _, openTransfer := range openTransfers {
+		fmt.Printf("Saving unmatched transfer from %s to %s on %s over %f as regular transaction\n", openTransfer.FromAccount, openTransfer.ToAccount, openTransfer.Date, openTransfer.Amount.GetFloat64())
+		_, err = ynab.queries.CreateTransaction(ynab.Context, openTransfer.CreateTransactionParams)
+		if err != nil {
+			return fmt.Errorf("could not save transaction %v: %w", openTransfer.CreateTransactionParams, err)
+		}
+
+	}
 	fmt.Printf("Imported %d transactions\n", count)
 
 	return nil

web/account.html

@@ -26,6 +26,9 @@
                 {{.CategoryGroup}} : {{.Category}}
             {{end}}
         </td>
+        <td>
+            {{if .GroupID.Valid}}☀{{end}}
+        </td>
         <td>
             <a href="/budget/{{$.Budget.ID}}/transaction/{{.ID}}">{{.Memo}}</a>
         </td>