Skip to content

feat: skip duplicates #105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
skynet2 merged 4 commits into from
Feb 21, 2026
Merged

feat: skip duplicates #105

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3766444
feat: add skip duplicates
skynet2 Feb 21, 2026
ca2b9e0
fix: lint
skynet2 Feb 21, 2026
0c3672d
fix: lint
skynet2 Feb 21, 2026
103b572
fix: new lint
skynet2 Feb 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions frontend/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions frontend/src/app/pages/transactions/transactions-import.component.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@

<p-checkbox id="chkbox3" [(ngModel)]="treatDatesAsUtc" [binary]="true" />
<label for="chkbox3">Treat Dates as UTC</label>

<p-checkbox id="chkbox4" [(ngModel)]="skipDuplicateReferenceCheck" [binary]="true" />
<label for="chkbox4">Skip Duplicate Reference Check</label>
</div>
@if (this.isRawTextImport()) {
<p-iftalabel>
Expand Down
5 changes: 4 additions & 1 deletion frontend/src/app/pages/transactions/transactions-import.component.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ export class TransactionsImportComponent {
public sources = EnumService.getImportTypes();
public skipRules: boolean = false;
public treatDatesAsUtc: boolean = false;
public skipDuplicateReferenceCheck: boolean = false;
public isLoading: boolean = false;
public stagingMode: boolean = true;

Expand Down Expand Up @@ -114,7 +115,8 @@ export class TransactionsImportComponent {
create(ParseTransactionsRequestSchema, {
content: contents,
source: this.selectedSource,
treatDatesAsUtc: this.treatDatesAsUtc
treatDatesAsUtc: this.treatDatesAsUtc,
skipDuplicateReferenceCheck: this.skipDuplicateReferenceCheck
})
);

Expand Down Expand Up @@ -177,6 +179,7 @@ export class TransactionsImportComponent {
create(ImportTransactionsRequestSchema, {
skipRules: this.skipRules,
treatDatesAsUtc: this.treatDatesAsUtc,
skipDuplicateReferenceCheck: this.skipDuplicateReferenceCheck,
source: this.selectedSource,
content: contents
})
Expand Down
6 changes: 3 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/ft-t/go-money
go 1.24.0

require (
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260104141704-8728d2c3d2bb.2
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.10-20260104141704-8728d2c3d2bb.1
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260221145631-b1f442f55195.2
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.11-20260221145631-b1f442f55195.1
connectrpc.com/connect v1.19.1
connectrpc.com/grpcreflect v1.3.0
github.com/DATA-DOG/go-sqlmock v1.5.2
Expand Down Expand Up @@ -40,7 +40,7 @@ require (
github.com/yuin/gopher-lua v1.1.1
golang.org/x/crypto v0.41.0
golang.org/x/net v0.43.0
google.golang.org/protobuf v1.36.10
google.golang.org/protobuf v1.36.11
gorm.io/driver/postgres v1.6.0
gorm.io/gorm v1.30.1
layeh.com/gopher-luar v1.0.11
Expand Down
12 changes: 6 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA=
al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260104141704-8728d2c3d2bb.2 h1:7kBnsqWCfiH07qhOTr5X0F8NFtYDRzNjAcSL9jQfxcE=
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260104141704-8728d2c3d2bb.2/go.mod h1:gEifYXu//7jMqINMfR3bowTg4x0QVNEeOPG0aPCHqfA=
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.10-20260104141704-8728d2c3d2bb.1 h1:zdfQtxzenede57tReYipiR3ZObDQnqujnnXAIlWRZsw=
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.10-20260104141704-8728d2c3d2bb.1/go.mod h1:1m2p+4J6zxArUgz9MC5m6ViEir1z82cOu9Ub9Pgwa2Q=
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260221145631-b1f442f55195.2 h1:RKXNh2DP9OU7NJO1zIE7FPr1Hteg/5n5cKk0y7gDx3o=
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260221145631-b1f442f55195.2/go.mod h1:VTTULP3xLDXiwW6AkXskKv7ZBgzqh4wgauKnA8Svumc=
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.11-20260221145631-b1f442f55195.1 h1:TCX0BllmWXkSzbWvPtsD2a7FBELmFhX5NIX1B5Vf6HI=
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.11-20260221145631-b1f442f55195.1/go.mod h1:h6BlZCpceFTRzghEP6LSlTeYkE05pzKfQCnRYsP4kDA=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14=
connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w=
Expand Down Expand Up @@ -408,8 +408,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
32 changes: 24 additions & 8 deletions pkg/importers/importer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package importers

import (
"context"
"fmt"
"sort"
"strings"

Expand Down Expand Up @@ -49,6 +50,7 @@ func NewImporter(
func (i *Importer) CheckDuplicates(
ctx context.Context,
requests []*transactionsv1.CreateTransactionRequest,
skipDuplicateRefCheck bool,
) ([]*DeduplicationItem, error) {
var allRefs []string
refToItem := map[string]*DeduplicationItem{}
Expand All @@ -67,13 +69,26 @@ func (i *Importer) CheckDuplicates(
return nil, errors.New("all transactions must have at least one reference number for deduplication")
}

for _, ref := range validRefs {
if existing, exists := refToItem[ref]; exists {
return nil, errors.Errorf("duplicate reference number found in import data: ref=%s. raw=%v", ref,
existing.CreateRequest.Notes)
for idx, ref := range validRefs {
if _, exists := refToItem[ref]; exists {
if !skipDuplicateRefCheck {
return nil, errors.Errorf("duplicate reference number found in import data: ref=%s. raw=%v", ref,
refToItem[ref].CreateRequest.Notes)
}

counter := 1
newRef := fmt.Sprintf("%s_%d", ref, counter)
for _, exists := refToItem[newRef]; exists; _, exists = refToItem[newRef] {
counter++
newRef = fmt.Sprintf("%s_%d", ref, counter)
}

validRefs[idx] = newRef
Comment on lines +79 to +86
Copy link
Copy Markdown

@gemini-code-assist gemini-code-assist Bot Feb 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

This logic for finding a unique reference number can be simplified and made more readable. The current for loop is a bit unconventional. A clearer approach would be to loop until a unique newRef is found.

Suggested change
counter := 1
newRef := fmt.Sprintf("%s_%d", ref, counter)
for _, exists := refToItem[newRef]; exists; _, exists = refToItem[newRef] {
counter++
newRef = fmt.Sprintf("%s_%d", ref, counter)
}
validRefs[idx] = newRef
var newRef string
for counter := 1; ; counter++ {
newRef = fmt.Sprintf("%s_%d", ref, counter)
if _, exists := refToItem[newRef]; !exists {
break
}
}
validRefs[idx] = newRef

}
}

req.InternalReferenceNumbers = validRefs

item := &DeduplicationItem{CreateRequest: req}
items = append(items, item)

Expand Down Expand Up @@ -115,9 +130,10 @@ func (i *Importer) Import(
req *importv1.ImportTransactionsRequest,
) (*importv1.ImportTransactionsResponse, error) {
parsed, err := i.ParseInternal(ctx, &importv1.ParseTransactionsRequest{
Content: req.Content,
Source: req.Source,
TreatDatesAsUtc: req.TreatDatesAsUtc,
Content: req.Content,
Source: req.Source,
TreatDatesAsUtc: req.TreatDatesAsUtc,
SkipDuplicateReferenceCheck: req.SkipDuplicateReferenceCheck,
})
if err != nil {
return nil, err
Expand Down Expand Up @@ -252,7 +268,7 @@ func (i *Importer) ParseInternal(
r.Extra["import_batch_id"] = batchID
}

items, err := i.CheckDuplicates(ctx, parsed.CreateRequests)
items, err := i.CheckDuplicates(ctx, parsed.CreateRequests, req.SkipDuplicateReferenceCheck)
if err != nil {
log.Error().
Err(err).
Expand Down
43 changes: 37 additions & 6 deletions pkg/importers/importer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,7 @@ func TestCheckDuplicates(t *testing.T) {
},
}

result, err := imp.CheckDuplicates(context.TODO(), requests)
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
assert.NoError(t, err)
assert.Len(t, result, 2)
assert.Equal(t, "Transaction 1", result[0].CreateRequest.Title)
Expand All @@ -530,7 +530,7 @@ func TestCheckDuplicates(t *testing.T) {
},
}

result, err := imp.CheckDuplicates(context.TODO(), requests)
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
assert.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "all transactions must have at least one reference number for deduplication")
Expand All @@ -551,7 +551,7 @@ func TestCheckDuplicates(t *testing.T) {
},
}

result, err := imp.CheckDuplicates(context.TODO(), requests)
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
assert.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "all transactions must have at least one reference number for deduplication")
Expand All @@ -572,7 +572,7 @@ func TestCheckDuplicates(t *testing.T) {
},
}

result, err := imp.CheckDuplicates(context.TODO(), requests)
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
assert.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "all transactions must have at least one reference number for deduplication")
Expand All @@ -597,12 +597,43 @@ func TestCheckDuplicates(t *testing.T) {
},
}

result, err := imp.CheckDuplicates(context.TODO(), requests)
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
assert.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "duplicate reference number found in import data: ref=ref_duplicate")
})

t.Run("skip duplicate reference check adds suffix", func(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()

impl := NewMockImplementation(ctrl)
impl.EXPECT().Type().Return(importv1.ImportSource_IMPORT_SOURCE_FIREFLY)
imp := importers.NewImporter(&importers.ImporterConfig{}, impl)

requests := []*transactionsv1.CreateTransactionRequest{
{
Title: "Transaction 1",
InternalReferenceNumbers: []string{"ref_duplicate"},
},
{
Title: "Transaction 2",
InternalReferenceNumbers: []string{"ref_duplicate"},
},
{
Title: "Transaction 3",
InternalReferenceNumbers: []string{"ref_duplicate"},
},
}

result, err := imp.CheckDuplicates(context.TODO(), requests, true)
assert.NoError(t, err)
assert.Len(t, result, 3)
assert.Equal(t, []string{"ref_duplicate"}, result[0].CreateRequest.InternalReferenceNumbers)
assert.Equal(t, []string{"ref_duplicate_1"}, result[1].CreateRequest.InternalReferenceNumbers)
assert.Equal(t, []string{"ref_duplicate_2"}, result[2].CreateRequest.InternalReferenceNumbers)
})

t.Run("db error on check", func(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
Expand All @@ -625,7 +656,7 @@ func TestCheckDuplicates(t *testing.T) {
},
}

result, err := imp.CheckDuplicates(ctx, requests)
result, err := imp.CheckDuplicates(ctx, requests, false)
assert.Error(t, err)
assert.Nil(t, result)
assert.Contains(t, err.Error(), "failed to check existing transactions")
Expand Down
Loading