From 2f3e4bc74849ba33812924938bfa3051f4903345 Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Sun, 9 Jan 2022 20:27:51 +0000 Subject: [PATCH 1/9] Add transfers to list and skip --- postgres/ynab-import.go | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/postgres/ynab-import.go b/postgres/ynab-import.go index cb7af94..766e668 100644 --- a/postgres/ynab-import.go +++ b/postgres/ynab-import.go @@ -125,6 +125,8 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error { return fmt.Errorf("could not read from tsv: %w", err) } + var openTransfers []CreateTransactionParams + count := 0 for _, record := range csvData[1:] { accountName := record[0] @@ -141,12 +143,6 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error { return fmt.Errorf("could not parse date %s: %w", dateString, err) } - payeeName := record[3] - payeeID, err := ynab.GetPayee(payeeName) - if err != nil { - return fmt.Errorf("could not get payee %s: %w", payeeName, err) - } - categoryGroup, categoryName := record[5], record[6] //also in 4 joined by : category, err := ynab.GetCategory(categoryGroup, categoryName) if err != nil { @@ -162,21 +158,39 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error { return fmt.Errorf("could not parse amount from (%s/%s): %w", inflow, outflow, err) } - //status := record[10] - transaction := CreateTransactionParams{ Date: date, Memo: memo, AccountID: account.ID, - PayeeID: payeeID, CategoryID: category, Amount: amount, } - _, err = ynab.queries.CreateTransaction(ynab.Context, transaction) - if err != nil { - return fmt.Errorf("could not save transaction %v: %w", transaction, err) + + payeeName := record[3] + if strings.HasPrefix(payeeName, "Transfer : ") { + // Transaction is a transfer to + transferToAccountName := payeeName[11:] + transferToAccount, err := ynab.GetAccount(transferToAccountName) + if err != nil { + return fmt.Errorf("Could not get transfer account %s: %w", transferToAccountName, err) + } + openTransfers = append(openTransfers, transaction) + fmt.Printf("Found transfer from %s to %s over %f\n", account.Name, transferToAccount.Name, amount.GetFloat64()) + } else { + payeeID, err := ynab.GetPayee(payeeName) + if err != nil { + return fmt.Errorf("could not get payee %s: %w", payeeName, err) + } + transaction.PayeeID = payeeID + + _, err = ynab.queries.CreateTransaction(ynab.Context, transaction) + if err != nil { + return fmt.Errorf("could not save transaction %v: %w", transaction, err) + } } + //status := record[10] + count++ } -- 2.47.2 From 951e827d204abc15e9b1fd8b272ae0703043d5d7 Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Sun, 9 Jan 2022 20:47:43 +0000 Subject: [PATCH 2/9] Add transfer_id --- postgres/models.go | 1 + postgres/schema/20220109_212800_add-transfer-id.sql | 6 ++++++ postgres/transactions.sql.go | 6 ++++-- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 postgres/schema/20220109_212800_add-transfer-id.sql diff --git a/postgres/models.go b/postgres/models.go index ed57572..00a569d 100644 --- a/postgres/models.go +++ b/postgres/models.go @@ -64,6 +64,7 @@ type Transaction struct { AccountID uuid.UUID CategoryID uuid.NullUUID PayeeID uuid.NullUUID + TransferID uuid.NullUUID } type TransactionsByMonth struct { diff --git a/postgres/schema/20220109_212800_add-transfer-id.sql b/postgres/schema/20220109_212800_add-transfer-id.sql new file mode 100644 index 0000000..1f7af36 --- /dev/null +++ b/postgres/schema/20220109_212800_add-transfer-id.sql @@ -0,0 +1,6 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN + transfer_id uuid NULL REFERENCES transactions (id); + +-- +goose Down +ALTER TABLE transactions DROP COLUMN transfer_id; \ No newline at end of file diff --git a/postgres/transactions.sql.go b/postgres/transactions.sql.go index dfd568e..ee110fa 100644 --- a/postgres/transactions.sql.go +++ b/postgres/transactions.sql.go @@ -14,7 +14,7 @@ const createTransaction = `-- name: CreateTransaction :one INSERT INTO transactions (date, memo, amount, account_id, payee_id, category_id) VALUES ($1, $2, $3, $4, $5, $6) -RETURNING id, date, memo, amount, account_id, category_id, payee_id +RETURNING id, date, memo, amount, account_id, category_id, payee_id, transfer_id ` type CreateTransactionParams struct { @@ -44,6 +44,7 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa &i.AccountID, &i.CategoryID, &i.PayeeID, + &i.TransferID, ) return i, err } @@ -74,7 +75,7 @@ func (q *Queries) DeleteTransaction(ctx context.Context, id uuid.UUID) error { } const getTransaction = `-- name: GetTransaction :one -SELECT id, date, memo, amount, account_id, category_id, payee_id FROM transactions +SELECT id, date, memo, amount, account_id, category_id, payee_id, transfer_id FROM transactions WHERE id = $1 ` @@ -89,6 +90,7 @@ func (q *Queries) GetTransaction(ctx context.Context, id uuid.UUID) (Transaction &i.AccountID, &i.CategoryID, &i.PayeeID, + &i.TransferID, ) return i, err } -- 2.47.2 From beff7afcf701f7483674a22234a8193d7c583888 Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Sun, 9 Jan 2022 22:29:47 +0000 Subject: [PATCH 3/9] Add group_id --- postgres/models.go | 2 +- postgres/queries/transactions.sql | 4 ++-- .../schema/20220109_212800_add-transfer-id.sql | 6 ------ postgres/schema/20220109_214000_add-group-id.sql | 6 ++++++ postgres/transactions.sql.go | 14 ++++++++------ 5 files changed, 17 insertions(+), 15 deletions(-) delete mode 100644 postgres/schema/20220109_212800_add-transfer-id.sql create mode 100644 postgres/schema/20220109_214000_add-group-id.sql diff --git a/postgres/models.go b/postgres/models.go index 00a569d..1e45f17 100644 --- a/postgres/models.go +++ b/postgres/models.go @@ -64,7 +64,7 @@ type Transaction struct { AccountID uuid.UUID CategoryID uuid.NullUUID PayeeID uuid.NullUUID - TransferID uuid.NullUUID + GroupID uuid.NullUUID } type TransactionsByMonth struct { diff --git a/postgres/queries/transactions.sql b/postgres/queries/transactions.sql index 2bef52a..3ea8ee8 100644 --- a/postgres/queries/transactions.sql +++ b/postgres/queries/transactions.sql @@ -4,8 +4,8 @@ WHERE id = $1; -- name: CreateTransaction :one INSERT INTO transactions -(date, memo, amount, account_id, payee_id, category_id) -VALUES ($1, $2, $3, $4, $5, $6) +(date, memo, amount, account_id, payee_id, category_id, group_id) +VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING *; -- name: UpdateTransaction :exec diff --git a/postgres/schema/20220109_212800_add-transfer-id.sql b/postgres/schema/20220109_212800_add-transfer-id.sql deleted file mode 100644 index 1f7af36..0000000 --- a/postgres/schema/20220109_212800_add-transfer-id.sql +++ /dev/null @@ -1,6 +0,0 @@ --- +goose Up -ALTER TABLE transactions ADD COLUMN - transfer_id uuid NULL REFERENCES transactions (id); - --- +goose Down -ALTER TABLE transactions DROP COLUMN transfer_id; \ No newline at end of file diff --git a/postgres/schema/20220109_214000_add-group-id.sql b/postgres/schema/20220109_214000_add-group-id.sql new file mode 100644 index 0000000..122eb02 --- /dev/null +++ b/postgres/schema/20220109_214000_add-group-id.sql @@ -0,0 +1,6 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN + group_id uuid NULL; + +-- +goose Down +ALTER TABLE transactions DROP COLUMN group_id; \ No newline at end of file diff --git a/postgres/transactions.sql.go b/postgres/transactions.sql.go index ee110fa..b74d94b 100644 --- a/postgres/transactions.sql.go +++ b/postgres/transactions.sql.go @@ -12,9 +12,9 @@ import ( const createTransaction = `-- name: CreateTransaction :one INSERT INTO transactions -(date, memo, amount, account_id, payee_id, category_id) -VALUES ($1, $2, $3, $4, $5, $6) -RETURNING id, date, memo, amount, account_id, category_id, payee_id, transfer_id +(date, memo, amount, account_id, payee_id, category_id, group_id) +VALUES ($1, $2, $3, $4, $5, $6, $7) +RETURNING id, date, memo, amount, account_id, category_id, payee_id, group_id ` type CreateTransactionParams struct { @@ -24,6 +24,7 @@ type CreateTransactionParams struct { AccountID uuid.UUID PayeeID uuid.NullUUID CategoryID uuid.NullUUID + GroupID uuid.NullUUID } func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (Transaction, error) { @@ -34,6 +35,7 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa arg.AccountID, arg.PayeeID, arg.CategoryID, + arg.GroupID, ) var i Transaction err := row.Scan( @@ -44,7 +46,7 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa &i.AccountID, &i.CategoryID, &i.PayeeID, - &i.TransferID, + &i.GroupID, ) return i, err } @@ -75,7 +77,7 @@ func (q *Queries) DeleteTransaction(ctx context.Context, id uuid.UUID) error { } const getTransaction = `-- name: GetTransaction :one -SELECT id, date, memo, amount, account_id, category_id, payee_id, transfer_id FROM transactions +SELECT id, date, memo, amount, account_id, category_id, payee_id, group_id FROM transactions WHERE id = $1 ` @@ -90,7 +92,7 @@ func (q *Queries) GetTransaction(ctx context.Context, id uuid.UUID) (Transaction &i.AccountID, &i.CategoryID, &i.PayeeID, - &i.TransferID, + &i.GroupID, ) return i, err } -- 2.47.2 From 2ec9c923dfb155b95d63c4f690fd6e69dd70b352 Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Sun, 9 Jan 2022 22:29:56 +0000 Subject: [PATCH 4/9] Implement matching --- postgres/ynab-import.go | 57 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/postgres/ynab-import.go b/postgres/ynab-import.go index 766e668..f976df4 100644 --- a/postgres/ynab-import.go +++ b/postgres/ynab-import.go @@ -113,6 +113,13 @@ func (ynab *YNABImport) ImportAssignments(r io.Reader) error { return nil } +type Transfer struct { + CreateTransactionParams + TransferToAccount *Account + FromAccount string + ToAccount string +} + // ImportTransactions expects a TSV-file as exported by YNAB in the following format: func (ynab *YNABImport) ImportTransactions(r io.Reader) error { @@ -125,7 +132,7 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error { return fmt.Errorf("could not read from tsv: %w", err) } - var openTransfers []CreateTransactionParams + var openTransfers []Transfer count := 0 for _, record := range csvData[1:] { @@ -174,8 +181,49 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error { if err != nil { return fmt.Errorf("Could not get transfer account %s: %w", transferToAccountName, err) } - openTransfers = append(openTransfers, transaction) - fmt.Printf("Found transfer from %s to %s over %f\n", account.Name, transferToAccount.Name, amount.GetFloat64()) + + transfer := Transfer{ + transaction, + transferToAccount, + accountName, + transferToAccountName, + } + + found := false + for i, openTransfer := range openTransfers { + if openTransfer.TransferToAccount.ID != transfer.AccountID { + continue + } + if openTransfer.AccountID != transfer.TransferToAccount.ID { + continue + } + if openTransfer.Amount.GetFloat64() != -1*transfer.Amount.GetFloat64() { + continue + } + + fmt.Printf("Matched transfers from %s to %s over %f\n", account.Name, transferToAccount.Name, amount.GetFloat64()) + openTransfers[i] = openTransfers[len(openTransfers)-1] + openTransfers = openTransfers[:len(openTransfers)-1] + found = true + + groupID := uuid.New() + transfer.GroupID = uuid.NullUUID{UUID: groupID, Valid: true} + openTransfer.GroupID = uuid.NullUUID{UUID: groupID, Valid: true} + + _, err = ynab.queries.CreateTransaction(ynab.Context, transfer.CreateTransactionParams) + if err != nil { + return fmt.Errorf("could not save transaction %v: %w", transaction, err) + } + _, err = ynab.queries.CreateTransaction(ynab.Context, openTransfer.CreateTransactionParams) + if err != nil { + return fmt.Errorf("could not save transaction %v: %w", transaction, err) + } + break + } + + if !found { + openTransfers = append(openTransfers, transfer) + } } else { payeeID, err := ynab.GetPayee(payeeName) if err != nil { @@ -194,6 +242,9 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error { count++ } + for _, openTransfer := range openTransfers { + fmt.Printf("Found transfer from %s to %s on %s over %f\n", openTransfer.FromAccount, openTransfer.ToAccount, openTransfer.Date, openTransfer.Amount.GetFloat64()) + } fmt.Printf("Imported %d transactions\n", count) return nil -- 2.47.2 From 3c17d674f98774b0d6d99d76318dea24b62828ed Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Sun, 9 Jan 2022 22:31:18 +0000 Subject: [PATCH 5/9] Fix migration --- ...{20220109_214000_add-group-id.sql => 0012_add-group-id.sql} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename postgres/schema/{20220109_214000_add-group-id.sql => 0012_add-group-id.sql} (55%) diff --git a/postgres/schema/20220109_214000_add-group-id.sql b/postgres/schema/0012_add-group-id.sql similarity index 55% rename from postgres/schema/20220109_214000_add-group-id.sql rename to postgres/schema/0012_add-group-id.sql index 122eb02..1bc265e 100644 --- a/postgres/schema/20220109_214000_add-group-id.sql +++ b/postgres/schema/0012_add-group-id.sql @@ -1,6 +1,5 @@ -- +goose Up -ALTER TABLE transactions ADD COLUMN - group_id uuid NULL; +ALTER TABLE transactions ADD COLUMN group_id uuid NULL; -- +goose Down ALTER TABLE transactions DROP COLUMN group_id; \ No newline at end of file -- 2.47.2 From ac27dc783e2cee46deaf83a98a6620c73ccd5c54 Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Mon, 10 Jan 2022 10:01:35 +0000 Subject: [PATCH 6/9] Also fetch GroupID and highlight groups in transactions-view --- postgres/queries/transactions.sql | 4 ++-- postgres/transactions.sql.go | 8 ++++++-- web/account.html | 3 +++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/postgres/queries/transactions.sql b/postgres/queries/transactions.sql index 3ea8ee8..3c4aa3f 100644 --- a/postgres/queries/transactions.sql +++ b/postgres/queries/transactions.sql @@ -23,7 +23,7 @@ DELETE FROM transactions WHERE id = $1; -- name: GetTransactionsForBudget :many -SELECT transactions.id, transactions.date, transactions.memo, transactions.amount, +SELECT transactions.id, transactions.date, transactions.memo, transactions.amount, transactions.group_id, accounts.name as account, COALESCE(payees.name, '') as payee, COALESCE(category_groups.name, '') as category_group, COALESCE(categories.name, '') as category FROM transactions INNER JOIN accounts ON accounts.id = transactions.account_id @@ -35,7 +35,7 @@ ORDER BY transactions.date DESC LIMIT 200; -- name: GetTransactionsForAccount :many -SELECT transactions.id, transactions.date, transactions.memo, transactions.amount, +SELECT transactions.id, transactions.date, transactions.memo, transactions.amount, transactions.group_id, accounts.name as account, COALESCE(payees.name, '') as payee, COALESCE(category_groups.name, '') as category_group, COALESCE(categories.name, '') as category FROM transactions INNER JOIN accounts ON accounts.id = transactions.account_id diff --git a/postgres/transactions.sql.go b/postgres/transactions.sql.go index b74d94b..e3fc971 100644 --- a/postgres/transactions.sql.go +++ b/postgres/transactions.sql.go @@ -132,7 +132,7 @@ func (q *Queries) GetTransactionsByMonthAndCategory(ctx context.Context, budgetI } const getTransactionsForAccount = `-- name: GetTransactionsForAccount :many -SELECT transactions.id, transactions.date, transactions.memo, transactions.amount, +SELECT transactions.id, transactions.date, transactions.memo, transactions.amount, transactions.group_id, accounts.name as account, COALESCE(payees.name, '') as payee, COALESCE(category_groups.name, '') as category_group, COALESCE(categories.name, '') as category FROM transactions INNER JOIN accounts ON accounts.id = transactions.account_id @@ -149,6 +149,7 @@ type GetTransactionsForAccountRow struct { Date time.Time Memo string Amount Numeric + GroupID uuid.NullUUID Account string Payee string CategoryGroup string @@ -169,6 +170,7 @@ func (q *Queries) GetTransactionsForAccount(ctx context.Context, accountID uuid. &i.Date, &i.Memo, &i.Amount, + &i.GroupID, &i.Account, &i.Payee, &i.CategoryGroup, @@ -188,7 +190,7 @@ func (q *Queries) GetTransactionsForAccount(ctx context.Context, accountID uuid. } const getTransactionsForBudget = `-- name: GetTransactionsForBudget :many -SELECT transactions.id, transactions.date, transactions.memo, transactions.amount, +SELECT transactions.id, transactions.date, transactions.memo, transactions.amount, transactions.group_id, accounts.name as account, COALESCE(payees.name, '') as payee, COALESCE(category_groups.name, '') as category_group, COALESCE(categories.name, '') as category FROM transactions INNER JOIN accounts ON accounts.id = transactions.account_id @@ -205,6 +207,7 @@ type GetTransactionsForBudgetRow struct { Date time.Time Memo string Amount Numeric + GroupID uuid.NullUUID Account string Payee string CategoryGroup string @@ -225,6 +228,7 @@ func (q *Queries) GetTransactionsForBudget(ctx context.Context, budgetID uuid.UU &i.Date, &i.Memo, &i.Amount, + &i.GroupID, &i.Account, &i.Payee, &i.CategoryGroup, diff --git a/web/account.html b/web/account.html index 71dceeb..b3af46f 100644 --- a/web/account.html +++ b/web/account.html @@ -26,6 +26,9 @@ {{.CategoryGroup}} : {{.Category}} {{end}} + + {{if .GroupID.Valid}}☀{{end}} + {{.Memo}} -- 2.47.2 From 49af9cd2ef9565b06e734e5d2e76d94ee47b5e9e Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Mon, 10 Jan 2022 10:01:47 +0000 Subject: [PATCH 7/9] Fix logging wrong objects --- postgres/ynab-import.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres/ynab-import.go b/postgres/ynab-import.go index f976df4..f7c1d61 100644 --- a/postgres/ynab-import.go +++ b/postgres/ynab-import.go @@ -212,11 +212,11 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error { _, err = ynab.queries.CreateTransaction(ynab.Context, transfer.CreateTransactionParams) if err != nil { - return fmt.Errorf("could not save transaction %v: %w", transaction, err) + return fmt.Errorf("could not save transaction %v: %w", transfer.CreateTransactionParams, err) } _, err = ynab.queries.CreateTransaction(ynab.Context, openTransfer.CreateTransactionParams) if err != nil { - return fmt.Errorf("could not save transaction %v: %w", transaction, err) + return fmt.Errorf("could not save transaction %v: %w", openTransfer.CreateTransactionParams, err) } break } -- 2.47.2 From 6fe30231d83b8ee3f9d3d314533f3e6862a4475a Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Mon, 10 Jan 2022 10:02:07 +0000 Subject: [PATCH 8/9] Save all unmatched transfers as regular transactions --- postgres/ynab-import.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/postgres/ynab-import.go b/postgres/ynab-import.go index f7c1d61..a0b6481 100644 --- a/postgres/ynab-import.go +++ b/postgres/ynab-import.go @@ -243,7 +243,12 @@ func (ynab *YNABImport) ImportTransactions(r io.Reader) error { } for _, openTransfer := range openTransfers { - fmt.Printf("Found transfer from %s to %s on %s over %f\n", openTransfer.FromAccount, openTransfer.ToAccount, openTransfer.Date, openTransfer.Amount.GetFloat64()) + fmt.Printf("Saving unmatched transfer from %s to %s on %s over %f as regular transaction\n", openTransfer.FromAccount, openTransfer.ToAccount, openTransfer.Date, openTransfer.Amount.GetFloat64()) + _, err = ynab.queries.CreateTransaction(ynab.Context, openTransfer.CreateTransactionParams) + if err != nil { + return fmt.Errorf("could not save transaction %v: %w", openTransfer.CreateTransactionParams, err) + } + } fmt.Printf("Imported %d transactions\n", count) -- 2.47.2 From a8edeaafa1b0c183f7b03ec9d2ff78d8f76d805a Mon Sep 17 00:00:00 2001 From: Jan Bader Date: Mon, 10 Jan 2022 10:14:10 +0000 Subject: [PATCH 9/9] Disable docker build for prs --- .drone.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.drone.yml b/.drone.yml index 0b32ea7..b07b184 100644 --- a/.drone.yml +++ b/.drone.yml @@ -22,6 +22,11 @@ steps: dockerfile: build/Dockerfile tags: - latest + when: + event: + exclude: + - pull_request + image_pull_secrets: - hub.javil.eu \ No newline at end of file -- 2.47.2