Skip to content

Commit

Permalink
DATA-3399: Dataset Data Remove Functions (#4780)
Browse files Browse the repository at this point in the history
  • Loading branch information
@vpandiarajan20
vpandiarajan20 authored Feb 11, 2025
1 parent 1762ee7 commit 17ecf94
Show file tree
Hide file tree
Showing 2 changed files with 106 additions and 32 deletions.
89 changes: 60 additions & 29 deletions cli/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ var dataTagByFilterFlags = append([]cli.Flag{
&cli.StringSliceFlag{
Name: dataFlagFilterTags,
Usage: "tags filter. " +
"accepts tagged for all tagged data, untagged for all untagged data, or a list of tags for all data matching any of the tags",
"accepts 'tagged' for all tagged data, 'untagged' for all untagged data, or a list of tags for all data matching any of the tags",
},
},
commonFilterFlags...)
Expand Down Expand Up @@ -1053,7 +1053,7 @@ var app = &cli.App{
},
&cli.StringSliceFlag{
Name: generalFlagTags,
Usage: "tags filter. accepts tagged for all tagged data, untagged for all untagged data, or a list of tags",
Usage: "tags filter. accepts 'tagged' for all tagged data, 'untagged' for all untagged data, or a list of tags",
},
}, commonFilterFlags...),
Action: createCommandWithT[dataExportBinaryArgs](DataExportBinaryAction),
Expand Down Expand Up @@ -1414,6 +1414,7 @@ var app = &cli.App{
UsageText: createUsageText("dataset data", nil, false, true),
HideHelpCommand: true,
Subcommands: []*cli.Command{
//nolint:dupl
{
Name: "add",
Usage: "adds binary data either by IDs or filter to dataset",
Expand Down Expand Up @@ -1452,7 +1453,7 @@ var app = &cli.App{
},
{
Name: "filter",
Usage: "adds binary data from the specified filter to dataset",
Usage: "adds binary data associated with a filter to a dataset",
UsageText: createUsageText("dataset data add filter", []string{datasetFlagDatasetID}, true, false),
Flags: append([]cli.Flag{
&cli.StringFlag{
Expand All @@ -1463,43 +1464,73 @@ var app = &cli.App{
&cli.StringSliceFlag{
Name: generalFlagTags,
Usage: "tags filter. " +
"accepts tagged for all tagged data, untagged for all untagged data, or a list of tags for all data matching any of the tags",
"accepts 'tagged' for all tagged data, 'untagged' for all untagged data, " +
"or a list of tags for all data matching any of the tags",
},
},
commonFilterFlags...),
Action: createCommandWithT[dataAddToDatasetByFilterArgs](DataAddToDatasetByFilter),
},
},
},
//nolint:dupl
{
Name: "remove",
Usage: "removes binary data with file IDs in a single org and location from dataset",
UsageText: createUsageText(
"dataset data remove", []string{datasetFlagDatasetID, generalFlagOrgID, dataFlagLocationID, dataFlagFileIDs}, false, false,
),
Flags: []cli.Flag{
&cli.StringFlag{
Name: datasetFlagDatasetID,
Usage: "dataset ID from which data will be removed",
Required: true,
},
&cli.StringFlag{
Name: generalFlagOrgID,
Usage: "org ID to which data belongs",
Required: true,
},
&cli.StringFlag{
Name: dataFlagLocationID,
Usage: "location ID to which data belongs",
Required: true,
Name: "remove",
Usage: "removes binary data either by IDs or filter from dataset",
UsageText: createUsageText("dataset data remove", nil, false, true),
HideHelpCommand: true,
Subcommands: []*cli.Command{
{
Name: "ids",
Usage: "removes binary data with file IDs in a single org and location from a dataset",
UsageText: createUsageText(
"dataset data remove ids", []string{datasetFlagDatasetID, generalFlagOrgID, dataFlagLocationID, dataFlagFileIDs}, false, false,
),
Flags: []cli.Flag{
&cli.StringFlag{
Name: datasetFlagDatasetID,
Usage: "dataset ID from which data will be removed",
Required: true,
},
&cli.StringFlag{
Name: generalFlagOrgID,
Usage: "org ID to which data belongs",
Required: true,
},
&cli.StringFlag{
Name: dataFlagLocationID,
Usage: "location ID to which data belongs",
Required: true,
},
&cli.StringSliceFlag{
Name: dataFlagFileIDs,
Usage: "file IDs of data belonging to specified org and location",
Required: true,
},
},
Action: createCommandWithT[dataRemoveFromDatasetArgs](DataRemoveFromDataset),
},
&cli.StringSliceFlag{
Name: dataFlagFileIDs,
Usage: "file IDs of data belonging to specified org and location",
Required: true,
{
Name: "filter",
Usage: "removes binary data associated with a filter from a dataset",
UsageText: createUsageText("dataset data remove filter", []string{datasetFlagDatasetID}, true, false),
Flags: append([]cli.Flag{
&cli.StringFlag{
Name: datasetFlagDatasetID,
Usage: "dataset ID from which data will be removed",
Required: true,
},
&cli.StringSliceFlag{
Name: generalFlagTags,
Usage: "tags filter. " +
"accepts 'tagged' for all tagged data, 'untagged' for all untagged data, " +
"or a list of tags for all data matching any of the tags",
},
},
commonFilterFlags...),
Action: createCommandWithT[dataRemoveFromDatasetByFilterArgs](DataRemoveFromDatasetByFilter),
},
},
Action: createCommandWithT[dataRemoveFromDatasetArgs](DataRemoveFromDataset),
},
},
},
Expand Down
49 changes: 46 additions & 3 deletions cli/data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -947,7 +947,7 @@ type dataAddToDatasetByIDsArgs struct {
FileIDs []string
}

// DataAddToDatasetByIDs is the corresponding action for 'data dataset add ids'.
// DataAddToDatasetByIDs is the corresponding action for 'dataset data add ids'.
func DataAddToDatasetByIDs(c *cli.Context, args dataAddToDatasetByIDsArgs) error {
client, err := newViamClient(c)
if err != nil {
Expand Down Expand Up @@ -986,7 +986,7 @@ type dataAddToDatasetByFilterArgs struct {
DatasetID string
}

// DataAddToDatasetByFilter is the corresponding action for 'data dataset add filter'.
// DataAddToDatasetByFilter is the corresponding action for 'dataset data add filter'.
func DataAddToDatasetByFilter(c *cli.Context, args dataAddToDatasetByFilterArgs) error {
client, err := newViamClient(c)
if err != nil {
Expand Down Expand Up @@ -1028,7 +1028,7 @@ type dataRemoveFromDatasetArgs struct {
FileIDs []string
}

// DataRemoveFromDataset is the corresponding action for 'data dataset remove'.
// DataRemoveFromDataset is the corresponding action for 'dataset data remove ids'.
func DataRemoveFromDataset(c *cli.Context, args dataRemoveFromDatasetArgs) error {
client, err := newViamClient(c)
if err != nil {
Expand Down Expand Up @@ -1064,6 +1064,49 @@ func (c *viamClient) dataRemoveFromDataset(datasetID, orgID, locationID string,
return nil
}

type dataRemoveFromDatasetByFilterArgs struct {
DatasetID string
}

// DataRemoveFromDatasetByFilter is the corresponding action for 'dataset data remove filter'.
func DataRemoveFromDatasetByFilter(c *cli.Context, args dataRemoveFromDatasetByFilterArgs) error {
client, err := newViamClient(c)
if err != nil {
return err
}
filter, err := createDataFilter(c)
if err != nil {
return err
}

// set the dataset ID in the filter, so that we do not attempt to remove data that is not in the dataset
filter.DatasetId = args.DatasetID

if err := client.dataRemoveFromDatasetByFilter(filter, args.DatasetID); err != nil {
return err
}
return nil
}

// dataRemoveFromDatasetByFilter removes binary data from the specified filter from dataset.
func (c *viamClient) dataRemoveFromDatasetByFilter(filter *datapb.Filter, datasetID string) error {
if err := c.ensureLoggedIn(); err != nil {
return err
}
parallelActions := uint(100)

return c.performActionOnBinaryDataFromFilter(
func(id *datapb.BinaryID) error {
_, err := c.dataClient.RemoveBinaryDataFromDatasetByIDs(c.c.Context,
&datapb.RemoveBinaryDataFromDatasetByIDsRequest{DatasetId: datasetID, BinaryIds: []*datapb.BinaryID{id}})
return err
},
filter, parallelActions,
func(i int32) {
printf(c.c.App.Writer, "Removed %d files from dataset ID %s", i, datasetID)
})
}

type dataConfigureDatabaseUserArgs struct {
OrgID string
Password string
Expand Down

0 comments on commit 17ecf94

Please sign in to comment.