diff --git a/.github/workflows/build-push-env-docker.yml b/.github/workflows/build-push-env-docker.yml index 1bf695e7ed..cdb88e96b7 100644 --- a/.github/workflows/build-push-env-docker.yml +++ b/.github/workflows/build-push-env-docker.yml @@ -44,7 +44,7 @@ jobs: - centos7 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index eb13a2db7b..a340145473 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -27,7 +27,7 @@ jobs: name: Module Labeler runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Assign GitHub labels uses: actions/labeler@v4 with: diff --git a/.github/workflows/lint_and_test_admin-cli.yml b/.github/workflows/lint_and_test_admin-cli.yml index 788ba8ba3e..4fef441bf0 100644 --- a/.github/workflows/lint_and_test_admin-cli.yml +++ b/.github/workflows/lint_and_test_admin-cli.yml @@ -41,7 +41,7 @@ jobs: name: Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v2 with: @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v2 with: diff --git a/.github/workflows/lint_and_test_collector.yml b/.github/workflows/lint_and_test_collector.yml index e21ede1a87..6fc8308c2f 100644 --- a/.github/workflows/lint_and_test_collector.yml +++ b/.github/workflows/lint_and_test_collector.yml @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Go uses: actions/setup-go@v2 with: @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 1 - name: Set up Go @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 1 - name: Set up Go diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml index 1fcd505c75..799263566a 100644 --- a/.github/workflows/lint_and_test_cpp.yaml +++ b/.github/workflows/lint_and_test_cpp.yaml @@ -53,7 +53,7 @@ jobs: name: Lint runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: clang-format run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . @@ -66,7 +66,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free Disk Space (Ubuntu) run: | .github/workflows/free_disk_space.sh @@ -97,7 +97,7 @@ jobs: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - name: Clone code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Rebuild thirdparty if needed uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Build Pegasus @@ -162,7 +162,7 @@ jobs: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} options: --cap-add=SYS_PTRACE steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download artifact uses: "./.github/actions/download_artifact" - name: Run server tests @@ -179,7 +179,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Rebuild thirdparty if needed uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Build Pegasus @@ -246,7 +246,7 @@ jobs: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} options: --cap-add=SYS_PTRACE steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download artifact uses: "./.github/actions/download_artifact" - name: Run server tests @@ -266,7 +266,7 @@ jobs: # container: # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # steps: -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v4 # - name: Rebuild thirdparty if needed # uses: "./.github/actions/rebuild_thirdparty_if_needed" # - name: Build Pegasus @@ -329,7 +329,7 @@ jobs: # image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} # options: --cap-add=SYS_PTRACE # steps: -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v4 # - name: Download artifact # uses: "./.github/actions/download_artifact" # - name: Run server tests @@ -346,7 +346,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Rebuild thirdparty if needed uses: "./.github/actions/rebuild_thirdparty_if_needed" # TODO(yingchun): Append "-m dsn_utils_tests" to the command if not needed to pack server or tools, for example, the dependencies are static linked. @@ -370,7 +370,7 @@ jobs: image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }} options: --cap-add=SYS_PTRACE steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download artifact uses: "./.github/actions/download_artifact" - name: Run server tests @@ -386,7 +386,7 @@ jobs: # Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md brew install ccache brew install openssl@1.1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup cache uses: actions/cache@v3 with: diff --git a/.github/workflows/lint_and_test_go-client.yml b/.github/workflows/lint_and_test_go-client.yml index 4e02501f14..691121b0f1 100644 --- a/.github/workflows/lint_and_test_go-client.yml +++ b/.github/workflows/lint_and_test_go-client.yml @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Go uses: actions/setup-go@v2 with: @@ -66,7 +66,7 @@ jobs: - name: Install thrift run: sudo apt-get install -y thrift-compiler - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Go uses: actions/setup-go@v2 with: @@ -91,7 +91,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: "./.github/actions/rebuild_thirdparty_if_needed" - uses: "./.github/actions/build_pegasus" - uses: "./.github/actions/upload_artifact" @@ -118,7 +118,7 @@ jobs: make install cd - && rm -rf thrift-${THRIFT_VERSION} v${THRIFT_VERSION}.tar.gz - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v2 with: diff --git a/.github/workflows/lint_and_test_java-client.yml b/.github/workflows/lint_and_test_java-client.yml index b71d1e021e..6d172bd910 100644 --- a/.github/workflows/lint_and_test_java-client.yml +++ b/.github/workflows/lint_and_test_java-client.yml @@ -39,7 +39,7 @@ jobs: name: Spotless runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-java@v1 with: java-version: 8 @@ -58,7 +58,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Rebuild thirdparty if needed uses: "./.github/actions/rebuild_thirdparty_if_needed" - name: Build Pegasus @@ -79,7 +79,7 @@ jobs: matrix: java: [ '8', '11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v2 with: path: ~/.m2/repository diff --git a/.github/workflows/lint_and_test_pegic.yml b/.github/workflows/lint_and_test_pegic.yml index 3788b44121..da94f9b4a8 100644 --- a/.github/workflows/lint_and_test_pegic.yml +++ b/.github/workflows/lint_and_test_pegic.yml @@ -41,7 +41,7 @@ jobs: name: Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v2 with: diff --git a/.github/workflows/lint_and_test_scala-client.yml b/.github/workflows/lint_and_test_scala-client.yml index b261902e73..29925d051b 100644 --- a/.github/workflows/lint_and_test_scala-client.yml +++ b/.github/workflows/lint_and_test_scala-client.yml @@ -39,7 +39,7 @@ jobs: name: Format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-java@v1 with: java-version: 8 @@ -57,7 +57,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: "./.github/actions/rebuild_thirdparty_if_needed" - uses: "./.github/actions/build_pegasus" - uses: "./.github/actions/upload_artifact" @@ -73,7 +73,7 @@ jobs: matrix: java: [ '8', '11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v2 with: path: ~/.m2/repository @@ -89,9 +89,9 @@ jobs: source /github/home/.sdkman/bin/sdkman-init.sh sdk install sbt sbt -V - - name: Recompile thrift + - name: Download thrift working-directory: ./java-client/scripts - run: ./recompile_thrift.sh + run: ./download_thrift.sh - name: Build Java client working-directory: ./java-client run: | diff --git a/.github/workflows/regular-build.yml b/.github/workflows/regular-build.yml index d4c3fb1820..4bc36a32cb 100644 --- a/.github/workflows/regular-build.yml +++ b/.github/workflows/regular-build.yml @@ -39,7 +39,7 @@ jobs: name: Lint Cpp runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: clang-format run: ./scripts/run-clang-format.py --clang-format-executable clang-format-14 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r . @@ -70,7 +70,7 @@ jobs: working-directory: /root/incubator-pegasus steps: - name: Clone Apache Pegasus Source - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Unpack prebuilt third-parties uses: "./.github/actions/unpack_prebuilt_thirdparties" - name: Build Pegasus @@ -85,7 +85,7 @@ jobs: # to generate code as well. The thrift-compiler version on ubuntu-20.04 is 0.13.0 run: sudo apt-get install -y thrift-compiler - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v2 with: @@ -128,7 +128,7 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} diff --git a/.github/workflows/standardization_lint.yaml b/.github/workflows/standardization_lint.yaml index 66bcd16276..a88f13e66a 100644 --- a/.github/workflows/standardization_lint.yaml +++ b/.github/workflows/standardization_lint.yaml @@ -47,14 +47,14 @@ jobs: name: Check Markdown links runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: gaurav-nelson/github-action-markdown-link-check@1.0.13 dockerfile_linter: name: Lint Dockerfile runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: hadolint/hadolint-action@v3.1.0 with: recursive: true @@ -65,7 +65,7 @@ jobs: runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check License Header uses: apache/skywalking-eyes@main env: diff --git a/.github/workflows/test_nodejs-client.yml b/.github/workflows/test_nodejs-client.yml index 371263e80e..e662e2458f 100644 --- a/.github/workflows/test_nodejs-client.yml +++ b/.github/workflows/test_nodejs-client.yml @@ -46,7 +46,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: "./.github/actions/rebuild_thirdparty_if_needed" - uses: "./.github/actions/build_pegasus" - uses: "./.github/actions/upload_artifact" @@ -58,7 +58,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install nodejs uses: actions/setup-node@v3 with: diff --git a/.github/workflows/test_python-client.yml b/.github/workflows/test_python-client.yml index c1eaaa7b70..f846be60f0 100644 --- a/.github/workflows/test_python-client.yml +++ b/.github/workflows/test_python-client.yml @@ -46,7 +46,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: "./.github/actions/rebuild_thirdparty_if_needed" - uses: "./.github/actions/build_pegasus" - uses: "./.github/actions/upload_artifact" @@ -58,7 +58,7 @@ jobs: container: image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.11' diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml index 59d0430ea2..a4ea746653 100644 --- a/.github/workflows/thirdparty-regular-push.yml +++ b/.github/workflows/thirdparty-regular-push.yml @@ -43,7 +43,7 @@ jobs: build_push_src_docker_images: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -78,7 +78,7 @@ jobs: - ubuntu2204 - centos7 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -114,7 +114,7 @@ jobs: - ubuntu2204 - centos7 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -148,7 +148,7 @@ jobs: osversion: - ubuntu2204 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx @@ -184,7 +184,7 @@ jobs: osversion: - ubuntu2204 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx diff --git a/nodejs-client/src/dsn/dsn_types.js b/nodejs-client/src/dsn/dsn_types.js index 46eb508ffd..4f2ef0a6f0 100644 --- a/nodejs-client/src/dsn/dsn_types.js +++ b/nodejs-client/src/dsn/dsn_types.js @@ -256,6 +256,41 @@ rpc_address.prototype.equals = function(other){ return false; }; +// TODO(yingchun): host_port is now just a place holder and not well implemented, need improve it +var host_port_type = { + HOST_TYPE_INVALID : 0, + HOST_TYPE_IPV4 : 1, + HOST_TYPE_GROUP : 2 +}; + +var host_port = function(args) { + this.host = null; + this.port = 0; + this.type = host_port_type.HOST_TYPE_INVALID; + if(args && args.host){ + this.host = args.host; + } + if(args && args.port){ + this.port = args.port; + } + if(args && args.type){ + this.type = args.type; + } +}; + +host_port.prototype = {}; +host_port.prototype.read = function(input){ + this.host = input.readBinary(); + this.port = input.readI16(); + this.type = input.readByte(); +}; + +host_port.prototype.write = function(output){ + output.writeBinary(this.host); + output.writeI16(this.port); + output.writeByte(this.type); +}; + //value, calculate by app_id and partition index var gpid = function(args) { this.value = 0; @@ -298,6 +333,7 @@ module.exports = { error_code : error_code, task_code : task_code, rpc_address : rpc_address, + host_port : host_port, gpid : gpid, }; diff --git a/python-client/pypegasus/base/ttypes.py b/python-client/pypegasus/base/ttypes.py index def909a0d6..6ed2e88dee 100644 --- a/python-client/pypegasus/base/ttypes.py +++ b/python-client/pypegasus/base/ttypes.py @@ -265,13 +265,13 @@ def __init__(self): def is_valid(self): return self.address == 0 - def from_string(self, host_port): - host, port = host_port.split(':') - self.address = socket.ntohl(struct.unpack("I", socket.inet_aton(host))[0]) + def from_string(self, ip_port): + ip, port = ip_port.split(':') + self.address = socket.ntohl(struct.unpack("I", socket.inet_aton(ip))[0]) self.address = (self.address << 32) + (int(port) << 16) + 1 # TODO why + 1? return True - def to_host_port(self): + def to_ip_port(self): s = [] address = self.address port = (address >> 16) & 0xFFFF @@ -305,6 +305,73 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) + +# TODO(yingchun): host_port is now just a place holder and not well implemented, need improve it +class host_port_types(Enum): + kHostTypeInvalid = 0 + kHostTypeIpv4 = 1 + kHostTypeGroup = 2 + + +class host_port: + + thrift_spec = ( + (1, TType.STRING, 'host', None, None, ), # 1 + (2, TType.I16, 'port', None, None, ), # 2 + (3, TType.I08, 'type', None, None, ), # 3 + ) + + def __init__(self): + self.host = "" + self.port = 0 + self.type = host_port_types.kHostTypeInvalid + + def is_valid(self): + return self.type != host_port_types.kHostTypeInvalid + + def from_string(self, host_port_str): + host_and_port = host_port_str.split(':') + if len(host_and_port) != 2: + return False + self.host = host_and_port[0] + self.port = int(host_and_port[1]) + # TODO(yingchun): Maybe it's not true, improve it + self.type = host_port_types.kHostTypeIpv4 + return True + + def to_host_port(self): + if not self.is_valid(): + return None, None + return self.host, self.port + + def read(self, iprot): + self.host = iprot.readString() + self.port = iprot.readI16() + self.type = iprot.readByte() + + def write(self, oprot): + oprot.writeString(self.host) + oprot.writeI16(self.port) + oprot.writeByte(self.type) + + def validate(self): + return + + def __hash__(self): + return hash(self.host) ^ self.port ^ self.type + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return other.__class__.__name__ == "host_port" and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class gpid: thrift_spec = ( diff --git a/python-client/pypegasus/pgclient.py b/python-client/pypegasus/pgclient.py index 457e99c8a1..2438483ba0 100644 --- a/python-client/pypegasus/pgclient.py +++ b/python-client/pypegasus/pgclient.py @@ -249,12 +249,12 @@ def __init__(self, table_name, timeout): def add_meta_server(self, meta_addr): rpc_addr = rpc_address() if rpc_addr.from_string(meta_addr): - host_port_list = meta_addr.split(':') - if not len(host_port_list) == 2: + ip_port = meta_addr.split(':') + if not len(ip_port) == 2: return False - host, port = host_port_list[0], int(host_port_list[1]) - self.addr_list.append((host, port)) + ip, port = ip_port[0], int(ip_port[1]) + self.addr_list.append((ip, port)) return True else: @@ -281,9 +281,9 @@ def got_results(self, res): def query(self): ds = [] - for (host, port) in self.addr_list: + for (ip, port) in self.addr_list: rpc_addr = rpc_address() - rpc_addr.from_string(host + ':' + str(port)) + rpc_addr.from_string(ip + ':' + str(port)) if rpc_addr in self.session_dict: self.session_dict[rpc_addr].close() @@ -294,7 +294,7 @@ def query(self): None, self, self.timeout - ).connectTCP(host, port, self.timeout) + ).connectTCP(ip, port, self.timeout) d.addCallbacks(self.got_conn, self.got_err) d.addCallbacks(self.query_one, self.got_err) ds.append(d) @@ -345,7 +345,7 @@ def update_cfg(self, resp): if rpc_addr in connected_rpc_addrs or rpc_addr.address == 0: continue - host, port = rpc_addr.to_host_port() + ip, port = rpc_addr.to_ip_port() if rpc_addr in self.session_dict: self.session_dict[rpc_addr].close() @@ -356,7 +356,7 @@ def update_cfg(self, resp): None, self.container, self.timeout - ).connectTCP(host, port, self.timeout) + ).connectTCP(ip, port, self.timeout) connected_rpc_addrs[rpc_addr] = 1 d.addCallbacks(self.got_conn, self.got_err) ds.append(d) @@ -642,8 +642,8 @@ def __init__(self, meta_addrs=None, table_name='', self.table = Table(table_name, self, timeout) self.meta_session_manager = MetaSessionManager(table_name, timeout) if isinstance(meta_addrs, list): - for host_port in meta_addrs: - self.meta_session_manager.add_meta_server(host_port) + for meta_addr in meta_addrs: + self.meta_session_manager.add_meta_server(meta_addr) PegasusHash.populate_table() self.timeout_times = 0 self.update_partition = False