broker.proto

// Copyright 2023 Ant Group Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

syntax = "proto3";

package scql.pb;

option go_package = "proto-gen/scql";
option java_package = "org.secretflow.scql";

import "api/common.proto";
import "api/status.proto";
import "api/core.proto";
import "api/interpreter.proto";
import "google/api/annotations.proto";
import "google/api/field_behavior.proto";
import "google/protobuf/timestamp.proto";
import "libspu/spu.proto";

// IntraBrokerService only accepts requests from within the domain.
service IntraBrokerService {
  // ===== DQL APIs =====
  // Only for select statement

  // DoQuery
  //
  // Run Query synchronously and return query result if the query completes
  // within a specified timeout
  rpc DoQuery(QueryRequest) returns (QueryResponse) {
    option (google.api.http) = {
      post: "/intra/query"
      body: "*"
    };
  }
  // SubmitQuery
  //
  // Run Query asynchronously
  rpc SubmitQuery(QueryRequest) returns (SubmitResponse) {
    option (google.api.http) = {
      post: "/intra/query/submit"
      body: "*"
    };
  }
  // FetchResult
  //
  // Fetch query result of asynchronous query
  rpc FetchResult(FetchResultRequest) returns (FetchResultResponse) {
    option (google.api.http) = {
      post: "/intra/query/fetch"
      body: "*"
    };
  }
  // ExplainQuery
  //
  // Get explaination of a query
  rpc ExplainQuery(ExplainQueryRequest) returns (ExplainQueryResponse) {
    option (google.api.http) = {
      post: "/intra/query/explain"
      body: "*"
    };
  }

  // CancelQuery
  //
  // Cancel asynchronous query
  // It will release the occupied local computing resources in time, but only
  // try to notify other members to release their corresponding resources
  rpc CancelQuery(CancelQueryRequest) returns (CancelQueryResponse) {
    option (google.api.http) = {
      post: "/intra/query/cancel"
      body: "*"
    };
  }

  // ===== Admin APIs =====

  // Project Related API

  // CreateProject
  //
  // Create a new Project and automatically become the Project member and
  // creator
  rpc CreateProject(CreateProjectRequest) returns (CreateProjectResponse) {
    option (google.api.http) = {
      post: "/intra/project/create"
      body: "*"
    };
  }
  rpc UpdateProject(UpdateProjectRequest) returns (UpdateProjectResponse);
  // ListProjects
  //
  // List All Projects that have created and joined
  rpc ListProjects(ListProjectsRequest) returns (ListProjectsResponse) {
    option (google.api.http) = {
      post: "/intra/project/list"
      body: "*"
    };
  }
  rpc ArchiveProject(ArchiveProjectRequest) returns (ArchiveProjectResponse);

  // InviteMember
  //
  // Invite another member to join the Project you created
  rpc InviteMember(InviteMemberRequest) returns (InviteMemberResponse) {
    option (google.api.http) = {
      post: "/intra/member/invite"
      body: "*"
    };
  }
  // ListInvitations
  //
  // List all invitations sent and received
  rpc ListInvitations(ListInvitationsRequest)
      returns (ListInvitationsResponse) {
    option (google.api.http) = {
      post: "/intra/invitation/list"
      body: "*"
    };
  }
  // ProcessInvitation
  //
  // Process the received invitation, specify it by invitation_id, choose to
  // join the corresponding project or reject it
  rpc ProcessInvitation(ProcessInvitationRequest)
      returns (ProcessInvitationResponse) {
    option (google.api.http) = {
      post: "/intra/invitation/process"
      body: "*"
    };
  }

  // Table related API

  // CreateTable
  //
  // Create a Table you owned in specified Project
  rpc CreateTable(CreateTableRequest) returns (CreateTableResponse) {
    option (google.api.http) = {
      post: "/intra/table/create"
      body: "*"
    };
  }
  // ListTables
  //
  // List all Tables in specified Project
  rpc ListTables(ListTablesRequest) returns (ListTablesResponse) {
    option (google.api.http) = {
      post: "/intra/table/list"
      body: "*"
    };
  }
  // DropTable
  //
  // Drop a Table you owned in specified Project, the relevant CCLs will be
  // automatically cleared
  rpc DropTable(DropTableRequest) returns (DropTableResponse) {
    option (google.api.http) = {
      post: "/intra/table/drop"
      body: "*"
    };
  }

  // CreateView
  //
  // Create a view of selection of table
  rpc CreateView(CreateViewRequest) returns (CreateViewResponse) {
    option (google.api.http) = {
      post: "/intra/view/create"
      body: "*"
    };
  }

  // ListViews
  //
  // List all views in specified Project
  rpc ListViews(ListViewsRequest) returns (ListViewsResponse) {
    option (google.api.http) = {
      post: "/intra/view/list",
      body: "*"
    };
  }

  // DropView
  //
  // Drop a view you owned in specified Project
  rpc DropView(DropViewRequest) returns (DropViewResponse) {
    option (google.api.http) = {
      post: "/intra/view/drop",
      body: "*"
    };
  }

  // CCL related

  // GrantCCL
  //
  // Grant CCLs of your Table to a specific member
  rpc GrantCCL(GrantCCLRequest) returns (GrantCCLResponse) {
    option (google.api.http) = {
      post: "/intra/ccl/grant"
      body: "*"
    };
  }
  // RevokeCCL
  //
  // Revoke the CCLs you have granted to the specified member
  rpc RevokeCCL(RevokeCCLRequest) returns (RevokeCCLResponse) {
    option (google.api.http) = {
      post: "/intra/ccl/revoke"
      body: "*"
    };
  }
  // ShowCCL
  //
  // Show CCLs in specified Project, supports specifying Tables, members
  rpc ShowCCL(ShowCCLRequest) returns (ShowCCLResponse) {
    option (google.api.http) = {
      post: "/intra/ccl/show"
      body: "*"
    };
  }

  // Other APIs

  // CheckAndUpdateStatus
  //
  // Check and update status for Projects
  rpc CheckAndUpdateStatus(CheckAndUpdateStatusRequest)
      returns (CheckAndUpdateStatusResponse) {
    option (google.api.http) = {
      post: "/intra/status/check_and_update"
      body: "*"
    };
  }
}

message QueryRequest {
  RequestHeader header = 1;
  // The ID of the Project in which to run the query
  string project_id = 2 [(google.api.field_behavior) = REQUIRED];
  // SCQL query to be run
  string query = 3 [(google.api.field_behavior) = REQUIRED];
  DebugOptions debug_opts = 4;
  // Dry run query, only takes effect in sync query API, naming DoQuery.
  // SubmitQuery API will report an error if dry_run is set to True.
  // Dry run only checks query syntax & CCL.
  bool dry_run = 5;

  JobConfig job_config = 6;
  // Specify the unique id for query job, by default leave it empty and use the
  // id generated by scql. currently only used when work with
  // [SECRETPAD](https://github.com/secretflow/secretpad)
  string job_id = 7;
}

message QueryResponse {
  Status status = 1;
  QueryResult result = 2;
}

message SubmitResponse {
  Status status = 1;
  // The unique identifier of the query job, used to obtain the job result
  // later.
  string job_id = 2;
}

message FetchResultRequest {
  // The unique identifier of the query job, obtained when SubmitQuery is called
  string job_id = 1 [(google.api.field_behavior) = REQUIRED];
}

// QueryResult represents the result of an executed SQL query.
message QueryResult {
  int64 affected_rows = 1;
  repeated SQLWarning warnings = 2;
  double cost_time_s = 3;
  // Output columns are used to store the result datas
  repeated Tensor out_columns = 1000;
}

message JobStatus {
  // A brief summary that describes the current state of the job.
  string summary = 1;
  JobProgress progress = 2;
}

message FetchResultResponse {
  Status status = 1;

  // Result of corresponding query, provided when status.code == Code.OK
  QueryResult result = 2;
  // JobStatus may be provided when result is not ready(status.code ==
  // Code.NOT_READY)
  JobStatus job_status = 3;
}

message ExplainQueryRequest {
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  string query = 2 [(google.api.field_behavior) = REQUIRED];
  JobConfig job_config = 3;
}

message ExplainQueryResponse {
  Status status = 1;
  ExplainInfo explain = 2;
}

message CancelQueryRequest {
  string job_id = 1 [(google.api.field_behavior) = REQUIRED];
}

message CancelQueryResponse {
  Status status = 1;
}

message CreateProjectRequest {
  // The unique identifier of the Project, if empty, broker will generate unique
  // project id
  string project_id = 1;
  // Project name, human readable
  string name = 2;
  string description = 3;
  // Runtime conf for Project
  ProjectConfig conf = 4 [(google.api.field_behavior) = REQUIRED];
};

message CreateProjectResponse {
  Status status = 1;
  // If Project id is empty in request, broker will generate one and return it
  // in response.
  string project_id = 2;
};

// Update Project Configurations
// Only project creater could update project settings
message UpdateProjectRequest {
  string project_id = 1;
  ProjectConfig conf = 2;
}

message UpdateProjectResponse {
  Status status = 1;
}

message ListProjectsRequest {
  // The ids of Projects to be listed, if empty, all Projects will be listed
  repeated string ids = 1;
}

message ListProjectsResponse {
  Status status = 1;
  repeated ProjectDesc projects = 2;
}

message ProjectDesc {
  string project_id = 1;
  string name = 2;
  string description = 3;
  ProjectConfig conf = 4;
  // Creator of the Project
  string creator = 5;
  repeated string members = 6;
  // created_at is the time the project was first created
  google.protobuf.Timestamp created_at = 7;
}

message ArchiveProjectRequest {
  string project_id = 1;
  string comment = 2;
}
message ArchiveProjectResponse {
  Status status = 1;
}

message ListInvitationsRequest {
  oneof filter {
    InvitationStatus status = 1;
    string inviter = 2;
  }
}

message ListInvitationsResponse {
  Status status = 1;
  repeated ProjectInvitation invitations = 2;
}

enum InvitationRespond {
  ACCEPT = 0;
  DECLINE = 1;
}

enum InvitationStatus {
  UNDECIDED = 0;
  ACCEPTED = 1;
  DECLINED = 2;
  INVALID = 3;
}

message ProcessInvitationRequest {
  // The id used to specify the Invitation, corresponding to the result of
  // ListInvitation
  uint64 invitation_id = 1 [(google.api.field_behavior) = REQUIRED];
  // Choose to accept or decline the Invitation
  InvitationRespond respond = 2 [(google.api.field_behavior) = REQUIRED];
  string respond_comment = 3;
}
message ProcessInvitationResponse {
  Status status = 1;
}

message ProjectInvitation {
  uint64 invitation_id = 1;
  ProjectDesc project = 2;
  string inviter = 3;
  string invitee = 4;
  string postscript = 5;
  InvitationStatus status = 6;
}

// Import physical table `ref_table` in Project `project_id` as `table_name`
// (-- Question: define db_typ and column dtype in enum? --)
message CreateTableRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // Table name
  string table_name = 2 [(google.api.field_behavior) = REQUIRED];
  // The refered physical table
  string ref_table = 3 [(google.api.field_behavior) = REQUIRED];
  // The db_type of ref_table, maybe 'MySQL/Postgres/csvdb/...'
  string db_type = 4;
  message ColumnDesc {
    // Column name
    string name = 1;
    // Column data type, see Enum DataType definition in "api/v1/column.proto"
    string dtype = 2;
  }
  // Columns choosed to refer from physical table
  repeated ColumnDesc columns = 5 [(google.api.field_behavior) = REQUIRED];
}

message CreateViewRequest {
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  string view_name = 2 [(google.api.field_behavior) = REQUIRED];
  // This field is used to store the content for defining a view.
  // It can be in the form of a CREATE VIEW statement or a direct SELECT query.
  string query = 3 [(google.api.field_behavior) = REQUIRED];
}

message CreateViewResponse {
  Status status = 1;
}

message CreateTableResponse {
  Status status = 1;
}

message ListTablesRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // The names of Tables to be listed, if empty, all Tables will be listed
  repeated string names = 2;
}

message ListTablesResponse {
  Status status = 1;
  repeated TableMeta tables = 2;
}

message ListViewsRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // The names of views to be listed, if empty, all views will be listed
  repeated string names = 2;
}

message ListViewsResponse {
  Status status = 1;
  repeated TableMeta views = 2;
}

message DropTableRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // The name of Table to be droped
  string table_name = 2 [(google.api.field_behavior) = REQUIRED];
}
message DropTableResponse {
  Status status = 1;
}

message DropViewRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // The name of view to be droped
  string view_name = 2 [(google.api.field_behavior) = REQUIRED];
}
message DropViewResponse {
  Status status = 1;
}

message GrantCCLRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // CCLs
  repeated ColumnControl column_control_list = 2
      [(google.api.field_behavior) = REQUIRED];
}
message GrantCCLResponse {
  Status status = 1;
}

message RevokeCCLRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // CCLs
  repeated ColumnControl column_control_list = 2
      [(google.api.field_behavior) = REQUIRED];
}

message RevokeCCLResponse {
  Status status = 1;
}

message ShowCCLRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // If tables is empty, it will show all Tables in the Project
  repeated string tables = 2;
  // If dest_parties is empty, it will show all members in the Project.
  // dest_party value could be `self`, or `others`, or some specific name.
  repeated string dest_parties = 3;
}

message ShowCCLResponse {
  Status status = 1;
  // CCLs
  repeated ColumnControl column_control_list = 2;
}

message ProjectConfig {
  // The spu runtime configuration.
  spu.pb.RuntimeConfig spu_runtime_cfg = 1;
  int64 session_expire_seconds = 2;
  int64 unbalance_psi_ratio_threshold = 3;
  int64 unbalance_psi_larger_party_rows_count_threshold = 4;
  int32 psi_curve_type = 5;
  int64 http_max_payload_size = 6;
  int64 link_recv_timeout_sec = 7;
  int64 link_throttle_window_size = 8;
  int64 link_chunked_send_parallel_size = 9;
  // use value in config file if not set
  optional uint64 group_by_threshold = 10;
  optional bool reveal_group_mark = 11;
  // reveal the number of elements in each group
  optional bool reveal_group_count = 12;
}

message JobConfig {
  // if the value is 0 which means user does not set it, it would fallback to
  // project default setting
  int64 session_expire_seconds = 1;
  // job's time_zone for session, default is empty which means use engine's
  // default time_zone
  string time_zone = 2;
  int64 unbalance_psi_ratio_threshold = 3;
  int64 unbalance_psi_larger_party_rows_count_threshold = 4;
  int32 psi_curve_type = 5;
  int64 http_max_payload_size = 6;
  int64 link_recv_timeout_sec = 7;
  int64 link_throttle_window_size = 8;
  int64 link_chunked_send_parallel_size = 9;
  bool enable_session_logger_separation = 10;
  PsiAlgorithmType psi_type = 11;
}

message InviteMemberRequest {
  // Project id
  string project_id = 1 [(google.api.field_behavior) = REQUIRED];
  // Party code of invitee
  string invitee = 2 [(google.api.field_behavior) = REQUIRED];
  string postscript = 3;
  enum InvitationMethod {
    // push invitation request to invitee's broker
    PUSH = 0;
    // Get invitation code, then share to invitee via email or IM
    // tools.
    // status: tobe implemented.
    PULL = 1;
  }
  InvitationMethod method = 4;
}

message InviteMemberResponse {
  Status status = 1;
  // Valid only when using PULL invitation method.
  // Format looks like URL parameters
  string invitation_code = 2;
}

message CheckAndUpdateStatusRequest {
  // The ids of Projects to be checked and updated, if empty, all Projects
  // will be processed
  repeated string project_ids = 1;
}
message CheckAndUpdateStatusResponse {
  Status status = 1;
  // Valid only when status.code() == Code::PROJECT_CONFLICT
  // maps from project_id to conflicts in the Project
  map<string, ProjectConflict> conflicts = 2;
}
message ProjectConflict {
  message ConflictItem {
    // Human-readable conflict details, e.g: "table 'ta' owner conflict: 'alice'
    // in party alice; 'bob' in party bob"
    string message = 1;
  }
  repeated ConflictItem items = 1;
}

// InterBrokerService defines a collection of APIs which are used to communicate
// with other p2p brokers.
service InterBrokerService {
  // Distribute query task to other participants
  rpc DistributeQuery(DistributeQueryRequest) returns (DistributeQueryResponse);

  rpc CancelQueryJob(CancelQueryJobRequest) returns (CancelQueryJobResponse);

  // Project related
  rpc InviteToProject(InviteToProjectRequest) returns (InviteToProjectResponse);
  rpc ReplyInvitation(ReplyInvitationRequest) returns (ReplyInvitationResponse);

  // sync new table & ccl settings
  rpc SyncInfo(SyncInfoRequest) returns (SyncInfoResponse);
  // ask information (project settings/table schema/ccl settings)
  rpc AskInfo(AskInfoRequest) returns (AskInfoResponse);
  // sync job running info. Currently ask endpoint of peer engine(not issuer),
  // for three parties job
  // warning: not need in two parties job
  rpc ExchangeJobInfo(ExchangeJobInfoRequest) returns (ExchangeJobInfoResponse);
}

// part of compile options which are nothing to do with safety
message RunningOptions {
  bool batched = 1;
}

message DistributeQueryRequest {
  RequestHeader header = 1;
  BrokerProtocolVersion client_protocol = 2;
  // client is issuer
  PartyId client_id = 3;
  bytes signature = 4;
  string project_id = 5;
  string job_id = 6;
  string query = 7;
  // SCQLEngine endpoint of client
  string engine_endpoint = 8;
  Checksum client_checksum = 9;
  Checksum server_checksum = 10;
  bool is_async = 11;
  DebugOptions debug_opts = 12;
  bool dry_run = 13;
  string time_zone = 14;
  JobConfig job_config = 15;
  RunningOptions running_opts = 16;
  google.protobuf.Timestamp created_at = 17;
}

message DistributeQueryResponse {
  Status status = 1;
  BrokerProtocolVersion server_protocol = 2;
  // SCQLEngine endpoint of server
  string engine_endpoint = 3;
  ChecksumCompareResult client_checksum_result = 4;
  ChecksumCompareResult server_checksum_result = 5;
  Checksum expected_server_checksum = 6;
}

message CancelQueryJobRequest {
  PartyId client_id = 1;
  bytes signature = 2;
  string job_id = 3;
  // cancel reason
  string reason = 4;
}
message CancelQueryJobResponse {
  Status status = 1;
}

message InviteToProjectRequest {
  PartyId client_id = 1;
  bytes signature = 2;
  ProjectDesc project = 3;
  // party code of inviter
  // Question: usually equal to client_id, still need this field?
  string inviter = 4;
  string invitation_code = 5;
}

message InviteToProjectResponse {
  Status status = 1;
}

message ReplyInvitationRequest {
  PartyId client_id = 1;
  bytes signature = 2;
  string project_id = 3;
  string invitation_code = 4;
  InvitationRespond respond = 5;
  string respond_comment = 6;
}

message ReplyInvitationResponse {
  Status status = 1;
  bytes project_info = 2;
}

message SyncInfoRequest {
  PartyId client_id = 1;
  bytes signature = 2;
  string project_id = 3;
  ChangeEntry change_entry = 4;
}

message ChangeEntry {
  enum Action {
    CreateTable = 0;
    DropTable = 1;
    GrantCCL = 2;
    RevokeCCL = 3;
    AddProjectMember = 4;
    UpdateProjectConf = 5;
    ArchiveProject = 6;
  }
  Action action = 1;
  bytes data = 2;
}

message SyncInfoResponse {
  Status status = 1;
}

message AskInfoRequest {
  PartyId client_id = 1;
  bytes signature = 2;
  repeated ResourceSpec resource_specs = 3;
}

message ResourceSpec {
  enum ResourceKind {
    // project information
    Project = 0;
    // table schema
    Table = 1;
    // all ccls for specific table
    CCL = 2;
    // all status managed by server party for specific project
    All = 3;
  }
  ResourceKind kind = 1;
  string project_id = 2;
  repeated string table_names = 3;
  repeated string dest_parties = 4;
}

message AskInfoResponse {
  Status status = 1;
  repeated bytes datas = 2;
}

message Checksum {
  // checksum of schemas
  bytes table_schema = 1;
  // checksum of ccl
  bytes ccl = 2;
}

message ExchangeJobInfoRequest {
  string project_id = 1;
  string job_id = 2;
  PartyId client_id = 3;
  bytes signature = 4;
  Checksum server_checksum = 5;
}

enum ChecksumCompareResult {
  EQUAL = 0;
  TABLE_SCHEMA_NOT_EQUAL = 1;
  CCL_NOT_EQUAL = 2;
  TABLE_CCL_NOT_EQUAL = 3;
}

message ExchangeJobInfoResponse {
  Status status = 1;
  // engine endpoint
  string endpoint = 2;
  ChecksumCompareResult server_checksum_result = 3;
  // used when checksum result was not equal
  Checksum expected_server_checksum = 4;
}

message TableMeta {
  string table_name = 1;
  string ref_table = 2;
  string db_type = 3;
  // party code of table owner
  string table_owner = 4;

  message Column {
    string name = 1;
    string dtype = 2;
  }
  repeated Column columns = 5;
  string select_string = 6;
}

message PrivacyPolicy {
  repeated ColumnControl column_control_list = 1;
}

message ColumnControl {
  ColumnDef col = 1;
  // the code of party that the constraint applies to.
  string party_code = 2;
  Constraint constraint = 3;
}

message ColumnDef {
  string column_name = 1;
  string table_name = 2;
}

enum BrokerProtocolVersion {
  BROKER_SERVICE_PROTOCOL_V1 = 0;
}

enum Constraint {
  UNKNOWN = 0;
  // The column can be revealed at party_code.
  PLAINTEXT = 1;
  // The column or any column produced by it can NOT be revealed at
  // party_code. It can be used as condition in JOIN, WHERE, and GROUP BY.
  ENCRYPTED_ONLY = 2;
  // The column can be revealed at party_code if it is used as join key
  PLAINTEXT_AFTER_JOIN = 3;
  // The column can be revealed at party_code if it is used as group by key
  PLAINTEXT_AFTER_GROUP_BY = 4;
  // The column can be revealed at party_code, if and only if
  // it is the result of the comparison function e.g. select compare1 >
  // compare2 from t group by zip_code
  PLAINTEXT_AFTER_COMPARE = 5;
  // The column can be revealed at party_code, if and only if
  // it has been through a N to 1 mapping such as
  //     - Aggregation operation such as sum/min/max/avg/count, e.g.
  //          select avg(score) from t group by zip_code
  //          select avg(score) from t
  PLAINTEXT_AFTER_AGGREGATE = 6;
  // The column can be revealed at party_code if it is used as join payload.
  // For example:
  //   select t2.id, t2.col1 from t1 inner join t2 on t1.id = t2.id;
  // In the above query, t2.col1 is used as join payload, t2.id is used as
  // join key.
  PLAINTEXT_AS_JOIN_PAYLOAD = 7;
  // The column is not able to be revealed at the party_code but the rank of
  // this column can be revealed at party_code
  // e.g.
  // SELECT ROW_NUMBER() OVER(PARTITITON BY department ORDER BY salary DESC)
  // AS rank in the query above the salary and department are not supposed
  // to be revealed to the party_code, however the rank of salary in this
  // department is visible
  REVEAL_RANK = 8;
}