From a93f01fc0815a000b033718c04a87613d0266d89 Mon Sep 17 00:00:00 2001 From: "Chorazewicz, Igor" Date: Tue, 2 Nov 2021 16:00:53 +0100 Subject: [PATCH 01/31] Run centos and debian workflows on push and PR Run tests on CI Run long tests (navy/bench) every day on CI Run CI on prebuild docker image Run only centos build on CI Update docker file used in CI Centos8 is EOL Disable failing clang-format-check Add extra param to build-package.sh Add scripts for rebuilding/pushing docker images Taken from: https://github.com/pmem/dev-utils-kit/commit/30794c3e1bbc9273e87da3e8f3ce7e5a2792b19e Extend CI to rebuild docker automatically Update build-cachelib-docker.yml Do not use shallow clone to make sure Docker rebuild logic works correctly. Added required packages to install Intel ittapi Update CI to use intel/CacheLib repo (#17) Add multi-tier navy benchmark and run it on CI - fix navy multi-tier config for NUMA bindings added code coverage support in CacheLib Adding libdml to CentOS docker image (#53) only exclude allocator-test-NavySetupTestm, shm-test-test_page_size tests added perf and numactl to docker packages --------------------------------------------- one large commit for all CI and code coverage see above for the change history. --- .../workflows/build-cachelib-centos-long.yml | 39 ++++++ .github/workflows/build-cachelib-debian.yml | 43 ++++++ .github/workflows/build-cachelib-docker.yml | 49 +++++++ .github/workflows/clang-format-check.yml | 2 +- cachelib/CMakeLists.txt | 5 + .../consistency/navy-multi-tier.json | 54 ++++++++ .../test_configs/consistency/navy.json | 4 +- contrib/build-package.sh | 8 +- docker/build.sh | 97 ++++++++++++++ docker/images/build-image.sh | 38 ++++++ docker/images/centos-8streams.Dockerfile | 24 ++++ docker/images/install-cachelib-deps.sh | 14 ++ docker/images/install-dsa-deps.sh | 23 ++++ docker/images/push-image.sh | 49 +++++++ docker/pull-or-rebuild-image.sh | 124 ++++++++++++++++++ docker/run-build.sh | 17 +++ docker/set-ci-vars.sh | 111 ++++++++++++++++ run_code_coverage.sh | 20 +++ run_tests.sh | 14 ++ 19 files changed, 728 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-cachelib-centos-long.yml create mode 100644 .github/workflows/build-cachelib-debian.yml create mode 100644 .github/workflows/build-cachelib-docker.yml create mode 100644 cachelib/cachebench/test_configs/consistency/navy-multi-tier.json create mode 100755 docker/build.sh create mode 100755 docker/images/build-image.sh create mode 100644 docker/images/centos-8streams.Dockerfile create mode 100755 docker/images/install-cachelib-deps.sh create mode 100755 docker/images/install-dsa-deps.sh create mode 100755 docker/images/push-image.sh create mode 100755 docker/pull-or-rebuild-image.sh create mode 100755 docker/run-build.sh create mode 100755 docker/set-ci-vars.sh create mode 100755 run_code_coverage.sh create mode 100755 run_tests.sh diff --git a/.github/workflows/build-cachelib-centos-long.yml b/.github/workflows/build-cachelib-centos-long.yml new file mode 100644 index 0000000000..92165f603b --- /dev/null +++ b/.github/workflows/build-cachelib-centos-long.yml @@ -0,0 +1,39 @@ +name: build-cachelib-centos-latest +on: + schedule: + - cron: '0 7 * * *' + +jobs: + build-cachelib-centos8-latest: + name: "CentOS/latest - Build CacheLib with all dependencies" + runs-on: ubuntu-latest + # Docker container image name + container: "centos:latest" + steps: + - name: "update packages" + run: dnf upgrade -y + - name: "install sudo,git" + run: dnf install -y sudo git cmake gcc + - name: "System Information" + run: | + echo === uname === + uname -a + echo === /etc/os-release === + cat /etc/os-release + echo === df -hl === + df -hl + echo === free -h === + free -h + echo === top === + top -b -n1 -1 -Eg || timeout 1 top -b -n1 + echo === env === + env + echo === gcc -v === + gcc -v + - name: "checkout sources" + uses: actions/checkout@v2 + - name: "build CacheLib using build script" + run: ./contrib/build.sh -j -v -T + - name: "run tests" + timeout-minutes: 60 + run: cd opt/cachelib/tests && ../../../run_tests.sh long diff --git a/.github/workflows/build-cachelib-debian.yml b/.github/workflows/build-cachelib-debian.yml new file mode 100644 index 0000000000..5bc3ad3c70 --- /dev/null +++ b/.github/workflows/build-cachelib-debian.yml @@ -0,0 +1,43 @@ +name: build-cachelib-debian-10 +on: + schedule: + - cron: '30 5 * * 0,3' + +jobs: + build-cachelib-debian-10: + name: "Debian/Buster - Build CacheLib with all dependencies" + runs-on: ubuntu-latest + # Docker container image name + container: "debian:buster-slim" + steps: + - name: "update packages" + run: apt-get update + - name: "upgrade packages" + run: apt-get -y upgrade + - name: "install sudo,git" + run: apt-get install -y sudo git procps + - name: "System Information" + run: | + echo === uname === + uname -a + echo === /etc/os-release === + cat /etc/os-release + echo === df -hl === + df -hl + echo === free -h === + free -h + echo === top === + top -b -n1 -1 -Eg || timeout 1 top -b -n1 ; true + echo === env === + env + echo === cc -v === + cc -v || true + echo === g++ -v === + g++ - || true + - name: "checkout sources" + uses: actions/checkout@v2 + - name: "build CacheLib using build script" + run: ./contrib/build.sh -j -v -T + - name: "run tests" + timeout-minutes: 60 + run: cd opt/cachelib/tests && ../../../run_tests.sh diff --git a/.github/workflows/build-cachelib-docker.yml b/.github/workflows/build-cachelib-docker.yml new file mode 100644 index 0000000000..be28bc233c --- /dev/null +++ b/.github/workflows/build-cachelib-docker.yml @@ -0,0 +1,49 @@ +name: build-cachelib-docker +on: + push: + pull_request: + +jobs: + build-cachelib-docker: + name: "CentOS/latest - Build CacheLib with all dependencies" + runs-on: ubuntu-latest + env: + REPO: cachelib + GITHUB_REPO: intel/CacheLib + CONTAINER_REG: ghcr.io/pmem/cachelib + CONTAINER_REG_USER: ${{ secrets.GH_CR_USER }} + CONTAINER_REG_PASS: ${{ secrets.GH_CR_PAT }} + FORCE_IMAGE_ACTION: ${{ secrets.FORCE_IMAGE_ACTION }} + HOST_WORKDIR: ${{ github.workspace }} + WORKDIR: docker + IMG_VER: devel + strategy: + matrix: + CONFIG: ["OS=centos OS_VER=8streams PUSH_IMAGE=1"] + steps: + - name: "System Information" + run: | + echo === uname === + uname -a + echo === /etc/os-release === + cat /etc/os-release + echo === df -hl === + df -hl + echo === free -h === + free -h + echo === top === + top -b -n1 -1 -Eg || timeout 1 top -b -n1 + echo === env === + env + echo === gcc -v === + gcc -v + - name: "checkout sources" + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Pull the image or rebuild and push it + run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh $FORCE_IMAGE_ACTION + + - name: Run the build + run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build.sh diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml index 26d942d182..54045f0a36 100644 --- a/.github/workflows/clang-format-check.yml +++ b/.github/workflows/clang-format-check.yml @@ -1,6 +1,6 @@ # From: https://github.com/marketplace/actions/clang-format-check#multiple-paths name: clang-format Check -on: [pull_request] +on: [] jobs: formatting-check: name: Formatting Check diff --git a/cachelib/CMakeLists.txt b/cachelib/CMakeLists.txt index 6be819974e..407342b581 100644 --- a/cachelib/CMakeLists.txt +++ b/cachelib/CMakeLists.txt @@ -85,6 +85,11 @@ set(CMAKE_MODULE_PATH set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED True) +if(COVERAGE_ENABLED) + # Add code coverage + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -fprofile-arcs -ftest-coverage") +endif() + # include(fb_cxx_flags) message(STATUS "Update CXXFLAGS: ${CMAKE_CXX_FLAGS}") diff --git a/cachelib/cachebench/test_configs/consistency/navy-multi-tier.json b/cachelib/cachebench/test_configs/consistency/navy-multi-tier.json new file mode 100644 index 0000000000..076550bc5c --- /dev/null +++ b/cachelib/cachebench/test_configs/consistency/navy-multi-tier.json @@ -0,0 +1,54 @@ +{ + "cache_config" : { + "cacheSizeMB" : 300, + "poolRebalanceIntervalSec" : 1, + "moveOnSlabRelease" : true, + + "cacheDir": "/tmp/mem-tier2", + "memoryTiers" : [ + { + "ratio": 1, + "memBindNodes": 0 + }, + { + "ratio": 1, + "memBindNodes": 0 + } + ], + + "numPools" : 2, + "poolSizes" : [0.5, 0.5], + "allocFactor" : 2.0, + "nvmCacheSizeMB" : 1024 + }, + "test_config" : + { + + "checkConsistency" : true, + + "numOps" : 60000, + "numThreads" : 20, + "numKeys" : 200000, + + + "keySizeRange" : [1, 8, 64], + "keySizeRangeProbability" : [0.5, 0.5], + + "valSizeRange" : [256, 1024, 4096, 8192], + "valSizeRangeProbability" : [0.2, 0.7, 0.1], + + "chainedItemLengthRange" : [1, 2, 4, 32], + "chainedItemLengthRangeProbability" : [0.8, 0.18, 0.02], + + "chainedItemValSizeRange" : [1, 128, 256, 1024, 4096, 20480], + "chainedItemValSizeRangeProbability" : [0.1, 0.1, 0.2, 0.3, 0.3], + + "getRatio" : 0.8, + "setRatio" : 0.1, + "delRatio" : 0.0, + "addChainedRatio" : 0.05, + "keyPoolDistribution": [0.5, 0.5], + "opPoolDistribution" : [0.5, 0.5] + } + +} diff --git a/cachelib/cachebench/test_configs/consistency/navy.json b/cachelib/cachebench/test_configs/consistency/navy.json index 73b016a50f..b95b056d31 100644 --- a/cachelib/cachebench/test_configs/consistency/navy.json +++ b/cachelib/cachebench/test_configs/consistency/navy.json @@ -14,8 +14,8 @@ "checkConsistency" : true, - "numOps" : 30000000, - "numThreads" : 40, + "numOps" : 600000, + "numThreads" : 20, "numKeys" : 200000, diff --git a/contrib/build-package.sh b/contrib/build-package.sh index 755933bd44..f0f3283df0 100755 --- a/contrib/build-package.sh +++ b/contrib/build-package.sh @@ -78,9 +78,8 @@ build_tests= show_help= many_jobs= verbose= -PREFIX="$PWD/opt/cachelib/" - -while getopts :BSdhijtvp: param +install_path= +while getopts :BSdhijtvI: param do case $param in i) install=yes ;; @@ -91,7 +90,7 @@ do v) verbose=yes ;; j) many_jobs=yes ;; t) build_tests=yes ;; - p) PREFIX=$OPTARG ;; + I) install_path=${OPTARG} ; install=yes ;; ?) die "unknown option. See -h for help." esac done @@ -281,6 +280,7 @@ test -d cachelib || die "expected 'cachelib' directory not found in $PWD" # After ensuring we are in the correct directory, set the installation prefix" +PREFIX=${install_path:-"$PWD/opt/cachelib/"} CMAKE_PARAMS="$CMAKE_PARAMS -DCMAKE_INSTALL_PREFIX=$PREFIX" CMAKE_PREFIX_PATH="$PREFIX/lib/cmake:$PREFIX/lib64/cmake:$PREFIX/lib:$PREFIX/lib64:$PREFIX:${CMAKE_PREFIX_PATH:-}" export CMAKE_PREFIX_PATH diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 0000000000..bb82f0142d --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2022, Intel Corporation + +# +# build.sh - runs a Docker container from a Docker image with environment +# prepared for running CacheLib builds and tests. It uses Docker image +# tagged as described in ./images/build-image.sh. +# +# Notes: +# - set env var 'HOST_WORKDIR' to where the root of this project is on the host machine, +# - set env var 'OS' and 'OS_VER' properly to a system/Docker you want to build this +# repo on (for proper values take a look at the list of Dockerfiles at the +# utils/docker/images directory in this repo), e.g. OS=ubuntu, OS_VER=20.04, +# - set env var 'CONTAINER_REG' to container registry address +# [and possibly user/org name, and package name], e.g. "/pmem/CacheLib", +# - set env var 'DNS_SERVER' if you use one, +# - set env var 'COMMAND' to execute specific command within Docker container or +# env var 'TYPE' to pick command based on one of the predefined types of build (see below). +# + +set -e + +source $(dirname ${0})/set-ci-vars.sh +IMG_VER=${IMG_VER:-devel} +TAG="${OS}-${OS_VER}-${IMG_VER}" +IMAGE_NAME=${CONTAINER_REG}:${TAG} +CONTAINER_NAME=CacheLib-${OS}-${OS_VER} +WORKDIR=/CacheLib # working dir within Docker container +SCRIPTSDIR=${WORKDIR}/docker + +if [[ -z "${OS}" || -z "${OS_VER}" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set " \ + "(e.g. OS=fedora, OS_VER=32)." + exit 1 +fi + +if [[ -z "${HOST_WORKDIR}" ]]; then + echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \ + "the root of this project on the host machine." + exit 1 +fi + +if [[ -z "${CONTAINER_REG}" ]]; then + echo "ERROR: CONTAINER_REG environment variable is not set " \ + "(e.g. \"//\")." + exit 1 +fi + +# Set command to execute in the Docker container +COMMAND="./run-build.sh"; +echo "COMMAND to execute within Docker container: ${COMMAND}" + +if [ -n "${DNS_SERVER}" ]; then DOCKER_OPTS="${DOCKER_OPTS} --dns=${DNS_SERVER}"; fi + +# Check if we are running on a CI (Travis or GitHub Actions) +[ -n "${GITHUB_ACTIONS}" -o -n "${TRAVIS}" ] && CI_RUN="YES" || CI_RUN="NO" + +# Do not allocate a pseudo-TTY if we are running on GitHub Actions +[ ! "${GITHUB_ACTIONS}" ] && DOCKER_OPTS="${DOCKER_OPTS} --tty=true" + + +echo "Running build using Docker image: ${IMAGE_NAME}" + +# Run a container with +# - environment variables set (--env) +# - host directory containing source mounted (-v) +# - working directory set (-w) +docker run --privileged=true --name=${CONTAINER_NAME} -i \ + ${DOCKER_OPTS} \ + --env http_proxy=${http_proxy} \ + --env https_proxy=${https_proxy} \ + --env TERM=xterm-256color \ + --env WORKDIR=${WORKDIR} \ + --env SCRIPTSDIR=${SCRIPTSDIR} \ + --env GITHUB_REPO=${GITHUB_REPO} \ + --env CI_RUN=${CI_RUN} \ + --env TRAVIS=${TRAVIS} \ + --env GITHUB_ACTIONS=${GITHUB_ACTIONS} \ + --env CI_COMMIT=${CI_COMMIT} \ + --env CI_COMMIT_RANGE=${CI_COMMIT_RANGE} \ + --env CI_BRANCH=${CI_BRANCH} \ + --env CI_EVENT_TYPE=${CI_EVENT_TYPE} \ + --env CI_REPO_SLUG=${CI_REPO_SLUG} \ + --env DOC_UPDATE_GITHUB_TOKEN=${DOC_UPDATE_GITHUB_TOKEN} \ + --env DOC_UPDATE_BOT_NAME=${DOC_UPDATE_BOT_NAME} \ + --env DOC_REPO_OWNER=${DOC_REPO_OWNER} \ + --env COVERITY_SCAN_TOKEN=${COVERITY_SCAN_TOKEN} \ + --env COVERITY_SCAN_NOTIFICATION_EMAIL=${COVERITY_SCAN_NOTIFICATION_EMAIL} \ + --env TEST_TIMEOUT=${TEST_TIMEOUT} \ + --env TZ='Europe/Warsaw' \ + --shm-size=4G \ + -v ${HOST_WORKDIR}:${WORKDIR} \ + -v /etc/localtime:/etc/localtime \ + -w ${SCRIPTSDIR} \ + ${IMAGE_NAME} ${COMMAND} + diff --git a/docker/images/build-image.sh b/docker/images/build-image.sh new file mode 100755 index 0000000000..985a6e0ff1 --- /dev/null +++ b/docker/images/build-image.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2021, Intel Corporation +# +# build-image.sh - prepares a Docker image with -based environment for +# testing (or dev) purpose, tagged with ${CONTAINER_REG}:${OS}-${OS_VER}-${IMG_VER}, +# according to the ${OS}-${OS_VER}.Dockerfile file located in the same directory. +# IMG_VER is a version of Docker image (it usually relates to project's release tag) +# and it defaults to "devel". +# + +set -e +IMG_VER=${IMG_VER:-devel} +TAG="${OS}-${OS_VER}-${IMG_VER}" + +if [[ -z "${OS}" || -z "${OS_VER}" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set " \ + "(e.g. OS=fedora, OS_VER=34)." + exit 1 +fi + +if [[ -z "${CONTAINER_REG}" ]]; then + echo "ERROR: CONTAINER_REG environment variable is not set " \ + "(e.g. \"//\")." + exit 1 +fi + +echo "Check if the file ${OS}-${OS_VER}.Dockerfile exists" +if [[ ! -f "${OS}-${OS_VER}.Dockerfile" ]]; then + echo "Error: ${OS}-${OS_VER}.Dockerfile does not exist." + exit 1 +fi + +echo "Build a Docker image tagged with: ${CONTAINER_REG}:${TAG}" +docker build -t ${CONTAINER_REG}:${TAG} \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + -f ${OS}-${OS_VER}.Dockerfile . diff --git a/docker/images/centos-8streams.Dockerfile b/docker/images/centos-8streams.Dockerfile new file mode 100644 index 0000000000..29752c5d98 --- /dev/null +++ b/docker/images/centos-8streams.Dockerfile @@ -0,0 +1,24 @@ +FROM quay.io/centos/centos:stream8 + +RUN dnf install -y \ +cmake \ +sudo \ +git \ +tzdata \ +vim \ +gdb \ +clang \ +python36 \ +glibc-devel.i686 \ +xmlto \ +uuid \ +libuuid-devel \ +json-c-devel \ +perf \ +numactl + +COPY ./install-cachelib-deps.sh ./install-cachelib-deps.sh +RUN ./install-cachelib-deps.sh + +COPY ./install-dsa-deps.sh ./install-dsa-deps.sh +RUN ./install-dsa-deps.sh diff --git a/docker/images/install-cachelib-deps.sh b/docker/images/install-cachelib-deps.sh new file mode 100755 index 0000000000..6d8fbdef7b --- /dev/null +++ b/docker/images/install-cachelib-deps.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2022, Intel Corporation + +git clone -b develop https://github.com/intel/CacheLib CacheLib + +./CacheLib/contrib/prerequisites-centos8.sh + +for pkg in zstd googleflags googlelog googletest sparsemap fmt folly fizz wangle fbthrift ; +do + sudo ./CacheLib/contrib/build-package.sh -j -I /opt/ "$pkg" +done + +rm -rf CacheLib diff --git a/docker/images/install-dsa-deps.sh b/docker/images/install-dsa-deps.sh new file mode 100755 index 0000000000..b4c62ecc93 --- /dev/null +++ b/docker/images/install-dsa-deps.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Copyright 2023, Intel Corporation + +# Install idxd-config +git clone https://github.com/intel/idxd-config.git +cd idxd-config +./autogen.sh +./configure CFLAGS='-g -O2' --prefix=/usr --sysconfdir=/etc --libdir=/usr/lib64 +make +make check +sudo make install +cd ../ +rm -rf idxd-config + +# Install DML Library +git clone --recursive https://github.com/intel/DML.git +cd DML +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo .. +cmake --build . --target install +cd ../../ +rm -rf DML diff --git a/docker/images/push-image.sh b/docker/images/push-image.sh new file mode 100755 index 0000000000..8f516b4205 --- /dev/null +++ b/docker/images/push-image.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2021, Intel Corporation + +# +# push-image.sh - pushes the Docker image tagged as described in +# ./build-image.sh, to the ${CONTAINER_REG}. +# +# The script utilizes ${CONTAINER_REG_USER} and ${CONTAINER_REG_PASS} variables to +# log in to the ${CONTAINER_REG}. The variables can be set in the CI's configuration +# for automated builds. +# + +set -e +IMG_VER=${IMG_VER:-devel} +TAG="${OS}-${OS_VER}-${IMG_VER}" + +if [[ -z "${OS}" || -z "${OS_VER}" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set " \ + "(e.g. OS=fedora, OS_VER=34)." + exit 1 +fi + +if [[ -z "${CONTAINER_REG}" ]]; then + echo "ERROR: CONTAINER_REG environment variable is not set " \ + "(e.g. \"//\")." + exit 1 +fi + +if [[ -z "${CONTAINER_REG_USER}" || -z "${CONTAINER_REG_PASS}" ]]; then + echo "ERROR: variables CONTAINER_REG_USER=\"${CONTAINER_REG_USER}\" and " \ + "CONTAINER_REG_PASS=\"${CONTAINER_REG_PASS}\"" \ + "have to be set properly to allow login to the Container Registry." + exit 1 +fi + +# Check if the image tagged with ${CONTAINER_REG}:${TAG} exists locally +if [[ ! $(docker images -a | awk -v pattern="^${CONTAINER_REG}:${TAG}\$" \ + '$1":"$2 ~ pattern') ]] +then + echo "ERROR: Docker image tagged ${CONTAINER_REG}:${TAG} does not exist locally." + exit 1 +fi + +echo "Log in to the Container Registry: ${CONTAINER_REG}" +echo "${CONTAINER_REG_PASS}" | docker login ghcr.io -u="${CONTAINER_REG_USER}" --password-stdin + +echo "Push the image to the Container Registry" +docker push ${CONTAINER_REG}:${TAG} diff --git a/docker/pull-or-rebuild-image.sh b/docker/pull-or-rebuild-image.sh new file mode 100755 index 0000000000..dcdcb40e8c --- /dev/null +++ b/docker/pull-or-rebuild-image.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2016-2021, Intel Corporation + +# +# pull-or-rebuild-image.sh - rebuilds the Docker image used in the +# current build (if necessary) or pulls it from the Container Registry. +# Docker image is tagged as described in docker/build-image.sh, +# but IMG_VER defaults in this script to "latest" (just in case it's +# used locally without building any images). +# +# If Docker was rebuilt and all requirements are fulfilled (more details in +# push_image function below) image will be pushed to the ${CONTAINER_REG}. +# +# The script rebuilds the Docker image if: +# 1. the Dockerfile for the current OS version (${OS}-${OS_VER}.Dockerfile) +# or any .sh script in the Dockerfiles directory were modified and committed, or +# 2. "rebuild" param was passed as a first argument to this script. +# +# The script pulls the Docker image if: +# 1. it does not have to be rebuilt (based on committed changes), or +# 2. "pull" param was passed as a first argument to this script. +# + +set -e + +source $(dirname ${0})/set-ci-vars.sh +IMG_VER=${IMG_VER:-latest} +TAG="${OS}-${OS_VER}-${IMG_VER}" +IMAGES_DIR_NAME=images +BASE_DIR=docker/${IMAGES_DIR_NAME} + +if [[ -z "${OS}" || -z "${OS_VER}" ]]; then + echo "ERROR: The variables OS and OS_VER have to be set properly " \ + "(eg. OS=fedora, OS_VER=34)." + exit 1 +fi + +if [[ -z "${CONTAINER_REG}" ]]; then + echo "ERROR: CONTAINER_REG environment variable is not set " \ + "(e.g. \"//\")." + exit 1 +fi + +function build_image() { + echo "Building the Docker image for the ${OS}-${OS_VER}.Dockerfile" + pushd ${IMAGES_DIR_NAME} + ./build-image.sh + popd +} + +function pull_image() { + echo "Pull the image '${CONTAINER_REG}:${TAG}' from the Container Registry." + docker pull ${CONTAINER_REG}:${TAG} +} + +function push_image { + # Check if the image has to be pushed to the Container Registry: + # - only upstream (not forked) repository, + # - stable-* or master branch, + # - not a pull_request event, + # - and PUSH_IMAGE flag was set for current build. + if [[ "${CI_REPO_SLUG}" == "${GITHUB_REPO}" \ + && (${CI_BRANCH} == develop || ${CI_BRANCH} == main) \ + && ${CI_EVENT_TYPE} != "pull_request" \ + && ${PUSH_IMAGE} == "1" ]] + then + echo "The image will be pushed to the Container Registry: ${CONTAINER_REG}" + pushd ${IMAGES_DIR_NAME} + ./push-image.sh + popd + else + echo "Skip pushing the image to the Container Registry." + fi +} + +# If "rebuild" or "pull" are passed to the script as param, force rebuild/pull. +if [[ "${1}" == "rebuild" ]]; then + build_image + push_image + exit 0 +elif [[ "${1}" == "pull" ]]; then + pull_image + exit 0 +fi + +# Determine if we need to rebuild the image or just pull it from +# the Container Registry, based on committed changes. +if [ -n "${CI_COMMIT_RANGE}" ]; then + commits=$(git rev-list ${CI_COMMIT_RANGE}) +else + commits=${CI_COMMIT} +fi + +if [[ -z "${commits}" ]]; then + echo "'commits' variable is empty. Docker image will be pulled." +fi + +echo "Commits in the commit range:" +for commit in ${commits}; do echo ${commit}; done + +echo "Files modified within the commit range:" +files=$(for commit in ${commits}; do git diff-tree --no-commit-id --name-only \ + -r ${commit}; done | sort -u) +for file in ${files}; do echo ${file}; done + +# Check if committed file modifications require the Docker image to be rebuilt +for file in ${files}; do + # Check if modified files are relevant to the current build + if [[ ${file} =~ ^(${BASE_DIR})\/(${OS})-(${OS_VER})\.Dockerfile$ ]] \ + || [[ ${file} =~ ^(${BASE_DIR})\/.*\.sh$ ]] + then + build_image + push_image + exit 0 + fi +done + +# Getting here means rebuilding the Docker image isn't required (based on changed files). +# Pull the image from the Container Registry or rebuild anyway, if pull fails. +if ! pull_image; then + build_image + push_image +fi diff --git a/docker/run-build.sh b/docker/run-build.sh new file mode 100755 index 0000000000..02c7caf731 --- /dev/null +++ b/docker/run-build.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2022, Intel Corporation + +set -e + +function sudo_password() { + echo ${USERPASS} | sudo -Sk $* +} + +cd .. +mkdir build +cd build +cmake ../cachelib -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=/opt -DCMAKE_BUILD_TYPE=Debug +sudo_password make install -j$(nproc) + +cd /opt/tests && $WORKDIR/run_tests.sh diff --git a/docker/set-ci-vars.sh b/docker/set-ci-vars.sh new file mode 100755 index 0000000000..f6f52132c8 --- /dev/null +++ b/docker/set-ci-vars.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2020-2021, Intel Corporation + +# +# set-ci-vars.sh -- set CI variables common for both: +# Travis and GitHub Actions CIs +# + +set -e + +function get_commit_range_from_last_merge { + # get commit id of the last merge + LAST_MERGE=$(git log --merges --pretty=%H -1) + LAST_COMMIT=$(git log --pretty=%H -1) + RANGE_END="HEAD" + if [ -n "${GITHUB_ACTIONS}" ] && [ "${GITHUB_EVENT_NAME}" == "pull_request" ] && [ "${LAST_MERGE}" == "${LAST_COMMIT}" ]; then + # GitHub Actions commits its own merge in case of pull requests + # so the first merge commit has to be skipped. + + LAST_COMMIT=$(git log --pretty=%H -2 | tail -n1) + LAST_MERGE=$(git log --merges --pretty=%H -2 | tail -n1) + # If still the last commit is a merge commit it means we're manually + # merging changes (probably back from stable branch). We have to use + # left parent of the merge and the current commit for COMMIT_RANGE. + if [ "${LAST_MERGE}" == "${LAST_COMMIT}" ]; then + LAST_MERGE=$(git log --merges --pretty=%P -2 | tail -n1 | cut -d" " -f1) + RANGE_END=${LAST_COMMIT} + fi + elif [ "${LAST_MERGE}" == "${LAST_COMMIT}" ] && + ([ "${TRAVIS_EVENT_TYPE}" == "push" ] || [ "${GITHUB_EVENT_NAME}" == "push" ]); then + # Other case in which last commit equals last merge, is when committing + # a manual merge. Push events don't set proper COMMIT_RANGE. + # It has to be then set: from merge's left parent to the current commit. + LAST_MERGE=$(git log --merges --pretty=%P -1 | cut -d" " -f1) + fi + if [ "${LAST_MERGE}" == "" ]; then + # possible in case of shallow clones + # or new repos with no merge commits yet + # - pick up the first commit + LAST_MERGE=$(git log --pretty=%H | tail -n1) + fi + COMMIT_RANGE="${LAST_MERGE}..${RANGE_END}" + # make sure it works now + if ! git rev-list ${COMMIT_RANGE} >/dev/null; then + COMMIT_RANGE="" + fi + echo ${COMMIT_RANGE} +} + +COMMIT_RANGE_FROM_LAST_MERGE=$(get_commit_range_from_last_merge) + +if [ -n "${TRAVIS}" ]; then + CI_COMMIT=${TRAVIS_COMMIT} + CI_COMMIT_RANGE="${TRAVIS_COMMIT_RANGE/.../..}" + CI_BRANCH=${TRAVIS_BRANCH} + CI_EVENT_TYPE=${TRAVIS_EVENT_TYPE} + CI_REPO_SLUG=${TRAVIS_REPO_SLUG} + + # CI_COMMIT_RANGE is usually invalid for force pushes - fix it when used + # with non-upstream repository + if [ -n "${CI_COMMIT_RANGE}" -a "${CI_REPO_SLUG}" != "${GITHUB_REPO}" ]; then + if ! git rev-list ${CI_COMMIT_RANGE}; then + CI_COMMIT_RANGE=${COMMIT_RANGE_FROM_LAST_MERGE} + fi + fi + + case "${TRAVIS_CPU_ARCH}" in + "amd64") + CI_CPU_ARCH="x86_64" + ;; + *) + CI_CPU_ARCH=${TRAVIS_CPU_ARCH} + ;; + esac + +elif [ -n "${GITHUB_ACTIONS}" ]; then + CI_COMMIT=${GITHUB_SHA} + CI_COMMIT_RANGE=${COMMIT_RANGE_FROM_LAST_MERGE} + CI_BRANCH=$(echo ${GITHUB_REF} | cut -d'/' -f3) + CI_REPO_SLUG=${GITHUB_REPOSITORY} + CI_CPU_ARCH="x86_64" # GitHub Actions supports only x86_64 + + case "${GITHUB_EVENT_NAME}" in + "schedule") + CI_EVENT_TYPE="cron" + ;; + *) + CI_EVENT_TYPE=${GITHUB_EVENT_NAME} + ;; + esac + +else + CI_COMMIT=$(git log --pretty=%H -1) + CI_COMMIT_RANGE=${COMMIT_RANGE_FROM_LAST_MERGE} + CI_CPU_ARCH="x86_64" +fi + +export CI_COMMIT=${CI_COMMIT} +export CI_COMMIT_RANGE=${CI_COMMIT_RANGE} +export CI_BRANCH=${CI_BRANCH} +export CI_EVENT_TYPE=${CI_EVENT_TYPE} +export CI_REPO_SLUG=${CI_REPO_SLUG} +export CI_CPU_ARCH=${CI_CPU_ARCH} + +echo CI_COMMIT=${CI_COMMIT} +echo CI_COMMIT_RANGE=${CI_COMMIT_RANGE} +echo CI_BRANCH=${CI_BRANCH} +echo CI_EVENT_TYPE=${CI_EVENT_TYPE} +echo CI_REPO_SLUG=${CI_REPO_SLUG} +echo CI_CPU_ARCH=${CI_CPU_ARCH} diff --git a/run_code_coverage.sh b/run_code_coverage.sh new file mode 100755 index 0000000000..7722e262bf --- /dev/null +++ b/run_code_coverage.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +#Build CacheLib with flag -DCOVERAGE_ENABLED=ON + +# Track coverage +lcov -c -i -b . -d . -o Coverage.baseline +./run_tests.sh +lcov -c -d . -b . -o Coverage.out +lcov -a Coverage.baseline -a Coverage.out -o Coverage.combined + +# Generate report +COVERAGE_DIR='coverage_report' +genhtml Coverage.combined -o ${COVERAGE_DIR} +COVERAGE_REPORT="${COVERAGE_DIR}.tgz" +tar -zcvf ${COVERAGE_REPORT} ${COVERAGE_DIR} +echo "Created coverage report ${COVERAGE_REPORT}" + +# Cleanup +rm Coverage.baseline Coverage.out Coverage.combined +rm -rf ${COVERAGE_DIR} diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000000..111e218333 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Newline separated list of tests to ignore +BLACKLIST="allocator-test-NavySetupTest +shm-test-test_page_size" + +if [ "$1" == "long" ]; then + find -type f -executable | grep -vF "$BLACKLIST" | xargs -n1 bash -c +else + find -type f \( -not -name "*bench*" -and -not -name "navy*" \) -executable | grep -vF "$BLACKLIST" | xargs -n1 bash -c +fi + +../bin/cachebench --json_test_config ../test_configs/consistency/navy.json +../bin/cachebench --json_test_config ../test_configs/consistency/navy-multi-tier.json From 2a8fa604aa69c5887780de350da9aae41b7bc13a Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Fri, 3 Feb 2023 16:02:50 -0800 Subject: [PATCH 02/31] Adds createPutToken and switches findEviction to utilize combined locking. --- cachelib/allocator/CacheAllocator-inl.h | 326 +++++++------------ cachelib/allocator/CacheAllocator.h | 34 +- cachelib/allocator/MM2Q.h | 1 + cachelib/allocator/tests/BaseAllocatorTest.h | 30 +- 4 files changed, 153 insertions(+), 238 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 92a04807a7..42dc5cb717 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -835,7 +835,9 @@ CacheAllocator::releaseBackToAllocator(Item& it, // If this chained item is marked as moving, we will not free it. // We must capture the moving state before we do the decRef when - // we know the item must still be valid + // we know the item must still be valid. Item cannot be marked as + // exclusive. Only parent can be marked as such and even parent needs + // to be unmark prior to calling releaseBackToAllocator. const bool wasMoving = head->isMoving(); XDCHECK(!head->isMarkedForEviction()); @@ -1139,7 +1141,7 @@ bool CacheAllocator::moveRegularItem(Item& oldItem, // it is unsafe to replace the old item with a new one, so we should // also abort. if (!accessContainer_->replaceIf(oldItem, *newItemHdl, - itemExclusivePredicate)) { + itemSlabMovePredicate)) { return false; } @@ -1192,14 +1194,14 @@ bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, return false; } - const auto parentKey = oldItem.getParentItem(compressor_).getKey(); - - // Grab lock to prevent anyone else from modifying the chain + auto& expectedParent = oldItem.getParentItem(compressor_); + const auto parentKey = expectedParent.getKey(); auto l = chainedItemLocks_.lockExclusive(parentKey); + // verify old item under the lock auto parentHandle = validateAndGetParentHandleForChainedMoveLocked(oldItem, parentKey); - if (!parentHandle) { + if (!parentHandle || &expectedParent != parentHandle.get()) { return false; } @@ -1241,6 +1243,14 @@ bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, return true; } +template +typename CacheAllocator::NvmCacheT::PutToken +CacheAllocator::createPutToken(Item& item) { + const bool evictToNvmCache = shouldWriteToNvmCache(item); + return evictToNvmCache ? nvmCache_->createPutToken(item.getKey()) + : typename NvmCacheT::PutToken{}; +} + template void CacheAllocator::unlinkItemForEviction(Item& it) { XDCHECK(it.isMarkedForEviction()); @@ -1286,12 +1296,9 @@ CacheAllocator::getNextCandidate(PoolId pid, ? &toRecycle_->asChainedItem().getParentItem(compressor_) : toRecycle_; - const bool evictToNvmCache = shouldWriteToNvmCache(*candidate_); - auto putToken = evictToNvmCache - ? nvmCache_->createPutToken(candidate_->getKey()) - : typename NvmCacheT::PutToken{}; + auto putToken = createPutToken(*candidate_); - if (evictToNvmCache && !putToken.isValid()) { + if (shouldWriteToNvmCache(*candidate_) && !putToken.isValid()) { stats_.evictFailConcurrentFill.inc(); ++itr; continue; @@ -1919,13 +1926,13 @@ std::vector CacheAllocator::dumpEvictionIterator( std::vector content; auto& mm = *mmContainers_[pid][cid]; - auto evictItr = mm.getEvictionIterator(); - size_t i = 0; - while (evictItr && i < numItems) { - content.push_back(evictItr->toString()); - ++evictItr; - ++i; - } + + mm.withEvictionIterator([&content, numItems](auto&& itr) { + while (itr && content.size() < numItems) { + content.push_back(itr->toString()); + ++itr; + } + }); return content; } @@ -2474,7 +2481,7 @@ bool CacheAllocator::moveForSlabRelease( bool isMoved = false; auto startTime = util::getCurrentTimeSec(); - WriteHandle newItemHdl = allocateNewItemForOldItem(oldItem); + WriteHandle newItemHdl{}; for (unsigned int itemMovingAttempts = 0; itemMovingAttempts < config_.movingTries; @@ -2490,8 +2497,15 @@ bool CacheAllocator::moveForSlabRelease( return true; } + throttleWith(throttler, [&] { + XLOGF(WARN, + "Spent {} seconds, slab release still trying to move Item: {}. " + "Pool: {}, Class: {}.", + util::getCurrentTimeSec() - startTime, oldItem.toString(), + ctx.getPoolId(), ctx.getClassId()); + }); + if (!newItemHdl) { - // try to allocate again if it previously wasn't successful newItemHdl = allocateNewItemForOldItem(oldItem); } @@ -2502,14 +2516,6 @@ bool CacheAllocator::moveForSlabRelease( break; } } - - throttleWith(throttler, [&] { - XLOGF(WARN, - "Spent {} seconds, slab release still trying to move Item: {}. " - "Pool: {}, Class: {}.", - util::getCurrentTimeSec() - startTime, oldItem.toString(), - ctx.getPoolId(), ctx.getClassId()); - }); } // Return false if we've exhausted moving tries. @@ -2531,6 +2537,8 @@ bool CacheAllocator::moveForSlabRelease( ctx.getPoolId(), ctx.getClassId()); }); } + auto ref = oldItem.unmarkMoving(); + XDCHECK_EQ(ref, 0); const auto allocInfo = allocator_->getAllocInfo(oldItem.getMemory()); allocator_->free(&oldItem); @@ -2541,10 +2549,10 @@ bool CacheAllocator::moveForSlabRelease( } template -typename CacheAllocator::ReadHandle +typename CacheAllocator::WriteHandle CacheAllocator::validateAndGetParentHandleForChainedMoveLocked( const ChainedItem& item, const Key& parentKey) { - ReadHandle parentHandle{}; + WriteHandle parentHandle{}; try { parentHandle = findInternal(parentKey); // If the parent is not the same as the parent of the chained item, @@ -2563,6 +2571,7 @@ CacheAllocator::validateAndGetParentHandleForChainedMoveLocked( template typename CacheAllocator::WriteHandle CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { + XDCHECK(oldItem.isMoving()); if (oldItem.isChainedItem()) { const auto& oldChainedItem = oldItem.asChainedItem(); const auto parentKey = oldChainedItem.getParentItem(compressor_).getKey(); @@ -2576,8 +2585,8 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { return {}; } - // Set up the destination for the move. Since oldChainedItem would be - // marked as moving, it won't be picked for eviction. + // Set up the destination for the move. Since oldChainedItem would + // be marked as moving, it won't be picked for eviction. auto newItemHdl = allocateChainedItemInternal(parentHandle, oldChainedItem.getSize()); if (!newItemHdl) { @@ -2588,7 +2597,7 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { auto parentPtr = parentHandle.getInternal(); XDCHECK_EQ(reinterpret_cast(parentPtr), reinterpret_cast( - &oldChainedItem.getParentItem(compressor_))); + &newItemHdl->asChainedItem().getParentItem(compressor_))); return newItemHdl; } @@ -2656,54 +2665,9 @@ void CacheAllocator::evictForSlabRelease( const SlabReleaseContext& ctx, Item& item, util::Throttler& throttler) { auto startTime = util::getCurrentTimeSec(); while (true) { + XDCHECK(item.isMoving()); stats_.numEvictionAttempts.inc(); - // if the item is already in a state where only the exclusive bit is set, - // nothing needs to be done. We simply need to call unmarkMoving and free - // the item. - if (item.isOnlyMoving()) { - item.unmarkMoving(); - const auto res = - releaseBackToAllocator(item, RemoveContext::kNormal, false); - XDCHECK(ReleaseRes::kReleased == res); - return; - } - - // Since we couldn't move, we now evict this item. Owning handle will be - // the item's handle for regular/normal items and will be the parent - // handle for chained items. - auto owningHandle = - item.isChainedItem() - ? evictChainedItemForSlabRelease(item.asChainedItem()) - : evictNormalItemForSlabRelease(item); - - // we managed to evict the corresponding owner of the item and have the - // last handle for the owner. - if (owningHandle) { - const auto allocInfo = - allocator_->getAllocInfo(static_cast(&item)); - if (owningHandle->hasChainedItem()) { - (*stats_.chainedItemEvictions)[allocInfo.poolId][allocInfo.classId] - .inc(); - } else { - (*stats_.regularItemEvictions)[allocInfo.poolId][allocInfo.classId] - .inc(); - } - - stats_.numEvictionSuccesses.inc(); - - // we have the last handle. no longer need to hold on to the exclusive bit - item.unmarkMoving(); - - // manually decrement the refcount to call releaseBackToAllocator - const auto ref = decRef(*owningHandle); - XDCHECK(ref == 0); - const auto res = releaseBackToAllocator(*owningHandle.release(), - RemoveContext::kEviction, false); - XDCHECK(res == ReleaseRes::kReleased); - return; - } - if (shutDownInProgress_) { item.unmarkMoving(); allocator_->abortSlabRelease(ctx); @@ -2725,146 +2689,102 @@ void CacheAllocator::evictForSlabRelease( .toString()) : ""); }); - } -} - -template -typename CacheAllocator::WriteHandle -CacheAllocator::evictNormalItemForSlabRelease(Item& item) { - XDCHECK(item.isMoving()); - if (item.isOnlyMoving()) { - return WriteHandle{}; - } - - auto predicate = [](const Item& it) { return it.getRefCount() == 0; }; - - const bool evictToNvmCache = shouldWriteToNvmCache(item); - auto token = evictToNvmCache ? nvmCache_->createPutToken(item.getKey()) - : typename NvmCacheT::PutToken{}; - - // We remove the item from both access and mm containers. It doesn't matter - // if someone else calls remove on the item at this moment, the item cannot - // be freed as long as it's marked for eviction. - auto handle = accessContainer_->removeIf(item, std::move(predicate)); + // if the item is already in a state where only the exclusive bit is set, + // nothing needs to be done. We simply need to call unmarkMoving and free + // the item. + if (item.isOnlyMoving()) { + item.unmarkMoving(); + const auto res = + releaseBackToAllocator(item, RemoveContext::kNormal, false); + XDCHECK(ReleaseRes::kReleased == res); + return; + } - if (!handle) { - return handle; - } + typename NvmCacheT::PutToken token; + Item* evicted; + if (item.isChainedItem()) { + auto& expectedParent = item.asChainedItem().getParentItem(compressor_); + const std::string parentKey = expectedParent.getKey().str(); + auto l = chainedItemLocks_.lockExclusive(parentKey); - XDCHECK_EQ(reinterpret_cast(handle.get()), - reinterpret_cast(&item)); - XDCHECK_EQ(1u, handle->getRefCount()); - removeFromMMContainer(item); + // check if the child is still in mmContainer and the expected parent is + // valid under the chained item lock. + if (expectedParent.getKey() != parentKey || !item.isInMMContainer() || + item.isOnlyMoving() || + &expectedParent != &item.asChainedItem().getParentItem(compressor_) || + !expectedParent.isAccessible() || !expectedParent.hasChainedItem()) { + continue; + } - // now that we are the only handle and we actually removed something from - // the RAM cache, we enqueue it to nvmcache. - if (evictToNvmCache && shouldWriteToNvmCacheExclusive(item)) { - nvmCache_->put(*handle, std::move(token)); - } + // search if the child is present in the chain + { + auto parentHandle = findInternal(parentKey); + if (!parentHandle || parentHandle != &expectedParent) { + continue; + } - return handle; -} + ChainedItem* head = nullptr; + { // scope for the handle + auto headHandle = findChainedItem(expectedParent); + head = headHandle ? &headHandle->asChainedItem() : nullptr; + } -template -typename CacheAllocator::WriteHandle -CacheAllocator::evictChainedItemForSlabRelease(ChainedItem& child) { - XDCHECK(child.isMoving()); - - // We have the child marked as moving, but dont know anything about the - // state of the parent. Unlike the case of regular eviction where we are - // sure that the child is inside the MMContainer, ensuring its parent is - // valid, we can not make any assumptions here. We try to find the parent - // first through the access container and then verify that the parent's - // chain points to the child before cleaning up the parent. If the parent - // was in the process of being re-allocated or child was being removed - // concurrently, we would synchronize here on one of the checks. - Item& expectedParent = child.getParentItem(compressor_); - - // Grab exclusive lock since we are modifying the chain. at this point, we - // dont know the state of the parent. so we need to do some validity checks - // after we have the chained item lock to ensure that we got the lock off of - // a valid state. - const std::string parentKey = expectedParent.getKey().str(); - auto l = chainedItemLocks_.lockExclusive(parentKey); + bool found = false; + while (head) { + if (head == &item) { + found = true; + break; + } + head = head->getNext(compressor_); + } - // check if the child is still in mmContainer and the expected parent is - // valid under the chained item lock. - if (expectedParent.getKey() != parentKey || !child.isInMMContainer() || - child.isOnlyMoving() || - &expectedParent != &child.getParentItem(compressor_) || - !expectedParent.isAccessible() || !expectedParent.hasChainedItem()) { - return {}; - } + if (!found) { + continue; + } + } - // search if the child is present in the chain - auto parentHandle = findInternal(parentKey); - if (!parentHandle || parentHandle != &expectedParent) { - return {}; - } + evicted = &expectedParent; - ChainedItem* head = nullptr; - { // scope for the handle - auto headHandle = findChainedItem(expectedParent); - head = headHandle ? &headHandle->asChainedItem() : nullptr; - } + token = createPutToken(*evicted); + if (evicted->markForEviction()) { + // unmark the child so it will be freed + item.unmarkMoving(); + unlinkItemForEviction(*evicted); + } else { + continue; + } + } else { + evicted = &item; - bool found = false; - while (head) { - if (head == &child) { - found = true; - break; + token = createPutToken(*evicted); + if (evicted->markForEvictionWhenMoving()) { + unlinkItemForEviction(*evicted); + } else { + continue; + } } - head = head->getNext(compressor_); - } - - if (!found) { - return {}; - } - - // if we found the child in the parent's chain, we remove it and ensure that - // the handle we obtained was the last one. Before that, create a put token - // to guard any racing cache find to avoid item re-appearing in NvmCache. - const bool evictToNvmCache = shouldWriteToNvmCache(expectedParent); - - auto token = evictToNvmCache - ? nvmCache_->createPutToken(expectedParent.getKey()) - : typename NvmCacheT::PutToken{}; - - if (!accessContainer_->removeIf(expectedParent, - parentEvictForSlabReleasePredicate)) { - return {}; - } - // at this point, we should be the last handle owner - XDCHECK_EQ(1u, parentHandle->getRefCount()); + if (token.isValid() && shouldWriteToNvmCacheExclusive(*evicted)) { + nvmCache_->put(*evicted, std::move(token)); + } - // We remove the parent from both access and mm containers. It doesn't - // matter if someone else calls remove on the parent at this moment, it - // cannot be freed since we hold an active item handle - removeFromMMContainer(*parentHandle); + const auto allocInfo = + allocator_->getAllocInfo(static_cast(evicted)); + if (evicted->hasChainedItem()) { + (*stats_.chainedItemEvictions)[allocInfo.poolId][allocInfo.classId].inc(); + } else { + (*stats_.regularItemEvictions)[allocInfo.poolId][allocInfo.classId].inc(); + } - // In case someone else had removed this chained item from its parent by now - // So we check again to see if it has been unlinked from its parent - if (!child.isInMMContainer() || child.isOnlyMoving()) { - return {}; - } + stats_.numEvictionSuccesses.inc(); - // check after removing from the MMContainer that the parent is still not - // being marked as moving. If parent is moving, it will release the child - // item and we will wait for that. - if (parentHandle->isMoving()) { - return {}; - } - - // now that we are the only handle and we actually removed something from - // the RAM cache, we enqueue it to nvmcache. - if (evictToNvmCache && shouldWriteToNvmCacheExclusive(*parentHandle)) { - DCHECK(parentHandle->hasChainedItem()); - nvmCache_->put(*parentHandle, std::move(token)); + XDCHECK(evicted->getRefCount() == 0); + const auto res = + releaseBackToAllocator(*evicted, RemoveContext::kEviction, false); + XDCHECK(res == ReleaseRes::kReleased); + return; } - - return parentHandle; } template diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 4c55496853..2124c4875e 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1458,17 +1458,17 @@ class CacheAllocator : public CacheBase { // @return handle to the parent item if the validations pass // otherwise, an empty Handle is returned. // - ReadHandle validateAndGetParentHandleForChainedMoveLocked( + WriteHandle validateAndGetParentHandleForChainedMoveLocked( const ChainedItem& item, const Key& parentKey); // Given an existing item, allocate a new one for the // existing one to later be moved into. // - // @param oldItem the item we want to allocate a new item for + // @param item reference to the item we want to allocate a new item for // // @return handle to the newly allocated item // - WriteHandle allocateNewItemForOldItem(const Item& oldItem); + WriteHandle allocateNewItemForOldItem(const Item& item); // internal helper that grabs a refcounted handle to the item. This does // not record the access to reflect in the mmContainer. @@ -1527,7 +1527,7 @@ class CacheAllocator : public CacheBase { // callback is responsible for copying the contents and fixing the semantics // of chained item. // - // @param oldItem Reference to the item being moved + // @param oldItem item being moved // @param newItemHdl Reference to the handle of the new item being moved into // // @return true If the move was completed, and the containers were updated @@ -1782,13 +1782,14 @@ class CacheAllocator : public CacheBase { // // // @param ctx slab release context - // @param item old item to be moved elsewhere + // @param oldItem old item to be moved elsewhere + // @param handle handle to the item or to it's parent (if chained) // @param throttler slow this function down as not to take too much cpu // // @return true if the item has been moved // false if we have exhausted moving attempts bool moveForSlabRelease(const SlabReleaseContext& ctx, - Item& item, + Item& oldItem, util::Throttler& throttler); // "Move" (by copying) the content in this item to another memory @@ -1811,18 +1812,7 @@ class CacheAllocator : public CacheBase { Item& item, util::Throttler& throttler); - // Helper function to evict a normal item for slab release - // - // @return last handle for corresponding to item on success. empty handle on - // failure. caller can retry if needed. - WriteHandle evictNormalItemForSlabRelease(Item& item); - - // Helper function to evict a child item for slab release - // As a side effect, the parent item is also evicted - // - // @return last handle to the parent item of the child on success. empty - // handle on failure. caller can retry. - WriteHandle evictChainedItemForSlabRelease(ChainedItem& item); + typename NvmCacheT::PutToken createPutToken(Item& item); // Helper function to remove a item if expired. // @@ -1945,18 +1935,14 @@ class CacheAllocator : public CacheBase { std::optional saveNvmCache(); void saveRamCache(); - static bool itemExclusivePredicate(const Item& item) { - return item.getRefCount() == 0; + static bool itemSlabMovePredicate(const Item& item) { + return item.isMoving() && item.getRefCount() == 0; } static bool itemExpiryPredicate(const Item& item) { return item.getRefCount() == 1 && item.isExpired(); } - static bool parentEvictForSlabReleasePredicate(const Item& item) { - return item.getRefCount() == 1 && !item.isMoving(); - } - std::unique_ptr createDeserializer(); // Execute func on each item. `func` can throw exception but must ensure diff --git a/cachelib/allocator/MM2Q.h b/cachelib/allocator/MM2Q.h index 982eca21f9..9c5cac834c 100644 --- a/cachelib/allocator/MM2Q.h +++ b/cachelib/allocator/MM2Q.h @@ -68,6 +68,7 @@ class MM2Q { enum LruType { Warm, WarmTail, Hot, Cold, ColdTail, NumTypes }; // Config class for MM2Q + // TODO: implement support for useCombinedLockForIterators struct Config { // Create from serialized config explicit Config(SerializationConfigType configState) diff --git a/cachelib/allocator/tests/BaseAllocatorTest.h b/cachelib/allocator/tests/BaseAllocatorTest.h index f503d59f61..3441815974 100644 --- a/cachelib/allocator/tests/BaseAllocatorTest.h +++ b/cachelib/allocator/tests/BaseAllocatorTest.h @@ -4098,15 +4098,16 @@ class BaseAllocatorTest : public AllocatorTest { // Check that item is in the expected container. bool findItem(AllocatorT& allocator, typename AllocatorT::Item* item) { auto& container = allocator.getMMContainer(*item); - auto itr = container.getEvictionIterator(); bool found = false; - while (itr) { - if (itr.get() == item) { - found = true; - break; + container.withEvictionIterator([&found, &item](auto&& itr) { + while (itr) { + if (itr.get() == item) { + found = true; + break; + } + ++itr; } - ++itr; - } + }); return found; } @@ -5494,8 +5495,12 @@ class BaseAllocatorTest : public AllocatorTest { ASSERT_TRUE(big->isInMMContainer()); auto& mmContainer = alloc.getMMContainer(*big); - auto itr = mmContainer.getEvictionIterator(); - ASSERT_EQ(big.get(), &(*itr)); + + typename AllocatorT::Item* evictionCandidate = nullptr; + mmContainer.withEvictionIterator( + [&evictionCandidate](auto&& itr) { evictionCandidate = itr.get(); }); + + ASSERT_EQ(big.get(), evictionCandidate); alloc.remove("hello"); } @@ -5509,8 +5514,11 @@ class BaseAllocatorTest : public AllocatorTest { ASSERT_TRUE(small2->isInMMContainer()); auto& mmContainer = alloc.getMMContainer(*small2); - auto itr = mmContainer.getEvictionIterator(); - ASSERT_EQ(small2.get(), &(*itr)); + + typename AllocatorT::Item* evictionCandidate = nullptr; + mmContainer.withEvictionIterator( + [&evictionCandidate](auto&& itr) { evictionCandidate = itr.get(); }); + ASSERT_EQ(small2.get(), evictionCandidate); alloc.remove("hello"); } From c3a4db97a631a1e36fd4548f97320c2c6c1d29b7 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Wed, 6 Jul 2022 10:15:17 +0000 Subject: [PATCH 03/31] Add memory usage statistics for allocation classes This includes printing: - allocSize - allocated memory size - memory usage fraction --- cachelib/allocator/Cache.h | 6 ++++++ cachelib/allocator/CacheAllocator-inl.h | 8 ++++++++ cachelib/allocator/CacheAllocator.h | 3 +++ cachelib/allocator/memory/MemoryAllocatorStats.h | 11 +++++++++++ cachelib/allocator/tests/CacheBaseTest.cpp | 1 + cachelib/cachebench/cache/Cache.h | 4 ++++ cachelib/cachebench/cache/CacheStats.h | 14 ++++---------- 7 files changed, 37 insertions(+), 10 deletions(-) diff --git a/cachelib/allocator/Cache.h b/cachelib/allocator/Cache.h index e225ba8a01..082db65f7a 100644 --- a/cachelib/allocator/Cache.h +++ b/cachelib/allocator/Cache.h @@ -102,6 +102,12 @@ class CacheBase { // @param poolId the pool id virtual PoolStats getPoolStats(PoolId poolId) const = 0; + // Get Allocation Class specific stats. + // + // @param poolId the pool id + // @param classId the class id + virtual ACStats getACStats(PoolId poolId, ClassId classId) const = 0; + // @param poolId the pool id virtual AllSlabReleaseEvents getAllSlabReleaseEvents(PoolId poolId) const = 0; diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 42dc5cb717..718a8f126b 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -2312,6 +2312,14 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { return ret; } +template +ACStats CacheAllocator::getACStats(PoolId poolId, + ClassId classId) const { + const auto& pool = allocator_->getPool(poolId); + const auto& ac = pool.getAllocationClass(classId); + return ac.getStats(); +} + template PoolEvictionAgeStats CacheAllocator::getPoolEvictionAgeStats( PoolId pid, unsigned int slabProjectionLength) const { diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 2124c4875e..38c5cb7990 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1185,6 +1185,9 @@ class CacheAllocator : public CacheBase { // return cache's memory usage stats CacheMemoryStats getCacheMemoryStats() const override final; + // return stats for Allocation Class + ACStats getACStats(PoolId pid, ClassId cid) const override final; + // return the nvm cache stats map util::StatsMap getNvmCacheStatsMap() const override final; diff --git a/cachelib/allocator/memory/MemoryAllocatorStats.h b/cachelib/allocator/memory/MemoryAllocatorStats.h index 74ebbe64dd..65d82e000d 100644 --- a/cachelib/allocator/memory/MemoryAllocatorStats.h +++ b/cachelib/allocator/memory/MemoryAllocatorStats.h @@ -54,6 +54,17 @@ struct ACStats { constexpr size_t getTotalFreeMemory() const noexcept { return Slab::kSize * freeSlabs + freeAllocs * allocSize; } + + constexpr double usageFraction() const noexcept { + if (usedSlabs == 0) + return 0.0; + + return activeAllocs / (usedSlabs * allocsPerSlab); + } + + constexpr size_t totalAllocatedSize() const noexcept { + return activeAllocs * allocSize; + } }; // structure to query stats corresponding to a MemoryPool diff --git a/cachelib/allocator/tests/CacheBaseTest.cpp b/cachelib/allocator/tests/CacheBaseTest.cpp index 928fcc0c67..f249786743 100644 --- a/cachelib/allocator/tests/CacheBaseTest.cpp +++ b/cachelib/allocator/tests/CacheBaseTest.cpp @@ -34,6 +34,7 @@ class CacheBaseTest : public CacheBase, public SlabAllocatorTestBase { bool isObjectCache() const override { return false; } const MemoryPool& getPool(PoolId) const override { return memoryPool_; } PoolStats getPoolStats(PoolId) const override { return PoolStats(); } + ACStats getACStats(PoolId, ClassId) const { return ACStats(); }; AllSlabReleaseEvents getAllSlabReleaseEvents(PoolId) const override { return AllSlabReleaseEvents{}; } diff --git a/cachelib/cachebench/cache/Cache.h b/cachelib/cachebench/cache/Cache.h index fa0a2ea556..65c70c30c1 100644 --- a/cachelib/cachebench/cache/Cache.h +++ b/cachelib/cachebench/cache/Cache.h @@ -325,6 +325,10 @@ class Cache { // return the stats for the pool. PoolStats getPoolStats(PoolId pid) const { return cache_->getPoolStats(pid); } + ACStats getACStats(PoolId pid, ClassId cid) const { + return cache_->getACStats(pid, cid); + } + // return the total number of inconsistent operations detected since start. unsigned int getInconsistencyCount() const { return inconsistencyCount_.load(std::memory_order_relaxed); diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index c673c4b406..daca4bc80c 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -165,7 +165,7 @@ struct Stats { foreachAC([&](auto pid, auto cid, auto stats) { auto [allocSizeSuffix, allocSize] = formatMemory(stats.allocSize); auto [memorySizeSuffix, memorySize] = - formatMemory(stats.activeAllocs * stats.allocSize); + formatMemory(stats.totalAllocatedSize()); out << folly::sformat("pid{:2} cid{:4} {:8.2f}{} memorySize: {:8.2f}{}", pid, cid, allocSize, allocSizeSuffix, memorySize, memorySizeSuffix) @@ -177,15 +177,9 @@ struct Stats { // If the pool is not full, extrapolate usageFraction for AC assuming it // will grow at the same rate. This value will be the same for all ACs. - double acUsageFraction; - if (poolUsageFraction[pid] < 1.0) { - acUsageFraction = poolUsageFraction[pid]; - } else if (stats.usedSlabs == 0) { - acUsageFraction = 0.0; - } else { - acUsageFraction = - stats.activeAllocs / (stats.usedSlabs * stats.allocsPerSlab); - } + auto acUsageFraction = (poolUsageFraction[pid] < 1.0) + ? poolUsageFraction[pid] + : stats.usageFraction(); out << folly::sformat( "pid{:2} cid{:4} {:8.2f}{} usageFraction: {:4.2f}", pid, cid, From 2529f0a215b2bde527547733b12bbaa40b56d962 Mon Sep 17 00:00:00 2001 From: "Chorazewicz, Igor" Date: Tue, 28 Sep 2021 15:11:07 +0200 Subject: [PATCH 04/31] Initial multi-tier support implementation (rebased with NUMA and cs part 2) fix for compressed ptr (upstream) -> compress from false to true --- cachelib/allocator/CacheAllocator-inl.h | 458 ++++++++++++------ cachelib/allocator/CacheAllocator.h | 106 ++-- cachelib/allocator/PoolOptimizer.cpp | 2 + cachelib/allocator/memory/MemoryAllocator.h | 7 + cachelib/allocator/memory/SlabAllocator.h | 17 +- .../allocator/tests/AllocatorResizeTest.h | 8 +- cachelib/allocator/tests/BaseAllocatorTest.h | 8 +- cachelib/allocator/tests/TestBase-inl.h | 4 +- 8 files changed, 412 insertions(+), 198 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 718a8f126b..5dd8a85a72 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -16,6 +16,8 @@ #pragma once +#include + namespace facebook { namespace cachelib { @@ -35,6 +37,7 @@ CacheAllocator::CacheAllocator(SharedMemNewT, Config config) template CacheAllocator::CacheAllocator(SharedMemAttachT, Config config) : CacheAllocator(InitMemType::kMemAttach, config) { + /* TODO - per tier? */ for (auto pid : *metadata_.compactCachePools()) { isCompactCachePool_[pid] = true; } @@ -67,12 +70,12 @@ CacheAllocator::CacheAllocator( : serialization::CacheAllocatorMetadata{}}, allocator_(initAllocator(type)), compactCacheManager_(type != InitMemType::kMemAttach - ? std::make_unique(*allocator_) - : restoreCCacheManager()), + ? std::make_unique(*allocator_[0] /* TODO: per tier */) + : restoreCCacheManager(0/* TODO: per tier */)), compressor_(createPtrCompressor()), mmContainers_(type == InitMemType::kMemAttach ? deserializeMMContainers(*deserializer_, compressor_) - : MMContainers{}), + : MMContainers{getNumTiers()}), accessContainer_(initAccessContainer( type, detail::kShmHashTableName, config.accessConfig)), chainedItemAccessContainer_( @@ -105,48 +108,86 @@ CacheAllocator::~CacheAllocator() { } template -ShmSegmentOpts CacheAllocator::createShmCacheOpts() { +ShmSegmentOpts CacheAllocator::createShmCacheOpts(TierId tid) { ShmSegmentOpts opts; opts.alignment = sizeof(Slab); // TODO: we support single tier so far - if (config_.memoryTierConfigs.size() > 1) { - throw std::invalid_argument("CacheLib only supports a single memory tier"); + if (config_.memoryTierConfigs.size() > 2) { + throw std::invalid_argument("CacheLib only supports two memory tiers"); } - opts.memBindNumaNodes = config_.memoryTierConfigs[0].getMemBind(); + opts.memBindNumaNodes = config_.memoryTierConfigs[tid].getMemBind(); return opts; } +template +std::vector> +CacheAllocator::createPrivateAllocator() { + std::vector> allocators; + + if (isOnShm_) + allocators.emplace_back(std::make_unique( + getAllocatorConfig(config_), + tempShm_->getAddr(), + config_.getCacheSize())); + else + allocators.emplace_back(std::make_unique( + getAllocatorConfig(config_), + config_.getCacheSize())); + + return allocators; +} + template std::unique_ptr -CacheAllocator::createNewMemoryAllocator() { +CacheAllocator::createNewMemoryAllocator(TierId tid) { return std::make_unique( getAllocatorConfig(config_), shmManager_ - ->createShm(detail::kShmCacheName, config_.getCacheSize(), - config_.slabMemoryBaseAddr, createShmCacheOpts()) + ->createShm(detail::kShmCacheName + std::to_string(tid), + config_.getCacheSize(), config_.slabMemoryBaseAddr, + createShmCacheOpts(tid)) .addr, config_.getCacheSize()); } template std::unique_ptr -CacheAllocator::restoreMemoryAllocator() { +CacheAllocator::restoreMemoryAllocator(TierId tid) { return std::make_unique( deserializer_->deserialize(), shmManager_ - ->attachShm(detail::kShmCacheName, config_.slabMemoryBaseAddr, - createShmCacheOpts()) - .addr, + ->attachShm(detail::kShmCacheName + std::to_string(tid), + config_.slabMemoryBaseAddr, createShmCacheOpts(tid)).addr, config_.getCacheSize(), config_.disableFullCoredump); } +template +std::vector> +CacheAllocator::createAllocators() { + std::vector> allocators; + for (int tid = 0; tid < getNumTiers(); tid++) { + allocators.emplace_back(createNewMemoryAllocator(tid)); + } + return allocators; +} + +template +std::vector> +CacheAllocator::restoreAllocators() { + std::vector> allocators; + for (int tid = 0; tid < getNumTiers(); tid++) { + allocators.emplace_back(restoreMemoryAllocator(tid)); + } + return allocators; +} + template std::unique_ptr -CacheAllocator::restoreCCacheManager() { +CacheAllocator::restoreCCacheManager(TierId tid) { return std::make_unique( deserializer_->deserialize(), - *allocator_); + *allocator_[tid]); } template @@ -238,21 +279,15 @@ void CacheAllocator::initWorkers() { } template -std::unique_ptr CacheAllocator::initAllocator( +std::vector> +CacheAllocator::initAllocator( InitMemType type) { if (type == InitMemType::kNone) { - if (isOnShm_ == true) { - return std::make_unique(getAllocatorConfig(config_), - tempShm_->getAddr(), - config_.getCacheSize()); - } else { - return std::make_unique(getAllocatorConfig(config_), - config_.getCacheSize()); - } + return createPrivateAllocator(); } else if (type == InitMemType::kMemNew) { - return createNewMemoryAllocator(); + return createAllocators(); } else if (type == InitMemType::kMemAttach) { - return restoreMemoryAllocator(); + return restoreAllocators(); } // Invalid type @@ -321,11 +356,12 @@ CacheAllocator::allocate(PoolId poolId, template typename CacheAllocator::WriteHandle -CacheAllocator::allocateInternal(PoolId pid, - typename Item::Key key, - uint32_t size, - uint32_t creationTime, - uint32_t expiryTime) { +CacheAllocator::allocateInternalTier(TierId tid, + PoolId pid, + typename Item::Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -334,13 +370,14 @@ CacheAllocator::allocateInternal(PoolId pid, const auto requiredSize = Item::getRequiredSize(key, size); // the allocation class in our memory allocator. - const auto cid = allocator_->getAllocationClassId(pid, requiredSize); + const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize); + // TODO: per-tier (*stats_.allocAttempts)[pid][cid].inc(); - void* memory = allocator_->allocate(pid, requiredSize); + void* memory = allocator_[tid]->allocate(pid, requiredSize); if (memory == nullptr) { - memory = findEviction(pid, cid); + memory = findEviction(tid, pid, cid); } WriteHandle handle; @@ -351,7 +388,7 @@ CacheAllocator::allocateInternal(PoolId pid, // for example. SCOPE_FAIL { // free back the memory to the allocator since we failed. - allocator_->free(memory); + allocator_[tid]->free(memory); }; handle = acquire(new (memory) Item(key, size, creationTime, expiryTime)); @@ -362,7 +399,7 @@ CacheAllocator::allocateInternal(PoolId pid, } } else { // failed to allocate memory. - (*stats_.allocFailures)[pid][cid].inc(); + (*stats_.allocFailures)[pid][cid].inc(); // TODO: per-tier // wake up rebalancer if (!config_.poolRebalancerDisableForcedWakeUp && poolRebalancer_) { poolRebalancer_->wakeUp(); @@ -379,6 +416,21 @@ CacheAllocator::allocateInternal(PoolId pid, return handle; } +template +typename CacheAllocator::WriteHandle +CacheAllocator::allocateInternal(PoolId pid, + typename Item::Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime) { + auto tid = 0; /* TODO: consult admission policy */ + for(TierId tid = 0; tid < getNumTiers(); ++tid) { + auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime); + if (handle) return handle; + } + return {}; +} + template typename CacheAllocator::WriteHandle CacheAllocator::allocateChainedItem(const ReadHandle& parent, @@ -409,21 +461,26 @@ CacheAllocator::allocateChainedItemInternal( // number of bytes required for this item const auto requiredSize = ChainedItem::getRequiredSize(size); - const auto pid = allocator_->getAllocInfo(parent->getMemory()).poolId; - const auto cid = allocator_->getAllocationClassId(pid, requiredSize); + // TODO: is this correct? + auto tid = getTierId(*parent); + const auto pid = allocator_[tid]->getAllocInfo(parent->getMemory()).poolId; + const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize); + + // TODO: per-tier? Right now stats_ are not used in any public periodic + // worker (*stats_.allocAttempts)[pid][cid].inc(); - void* memory = allocator_->allocate(pid, requiredSize); + void* memory = allocator_[tid]->allocate(pid, requiredSize); if (memory == nullptr) { - memory = findEviction(pid, cid); + memory = findEviction(tid, pid, cid); } if (memory == nullptr) { (*stats_.allocFailures)[pid][cid].inc(); return WriteHandle{}; } - SCOPE_FAIL { allocator_->free(memory); }; + SCOPE_FAIL { allocator_[tid]->free(memory); }; auto child = acquire( new (memory) ChainedItem(compressor_.compress(parent.getInternal()), size, @@ -732,8 +789,8 @@ CacheAllocator::releaseBackToAllocator(Item& it, throw std::runtime_error( folly::sformat("cannot release this item: {}", it.toString())); } - - const auto allocInfo = allocator_->getAllocInfo(it.getMemory()); + const auto tid = getTierId(it); + const auto allocInfo = allocator_[tid]->getAllocInfo(it.getMemory()); if (ctx == RemoveContext::kEviction) { const auto timeNow = util::getCurrentTimeSec(); @@ -757,8 +814,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, folly::sformat("Can not recycle a chained item {}, toRecyle", it.toString(), toRecycle->toString())); } - - allocator_->free(&it); + allocator_[tid]->free(&it); return ReleaseRes::kReleased; } @@ -827,7 +883,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, auto next = head->getNext(compressor_); const auto childInfo = - allocator_->getAllocInfo(static_cast(head)); + allocator_[tid]->getAllocInfo(static_cast(head)); (*stats_.fragmentationSize)[childInfo.poolId][childInfo.classId].sub( util::getFragmentation(*this, *head)); @@ -863,7 +919,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, XDCHECK(ReleaseRes::kReleased != res); res = ReleaseRes::kRecycled; } else { - allocator_->free(head); + allocator_[tid]->free(head); } } @@ -878,7 +934,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, res = ReleaseRes::kRecycled; } else { XDCHECK(it.isDrained()); - allocator_->free(&it); + allocator_[tid]->free(&it); } return res; @@ -1267,13 +1323,14 @@ void CacheAllocator::unlinkItemForEviction(Item& it) { template std::pair::Item*, typename CacheAllocator::Item*> -CacheAllocator::getNextCandidate(PoolId pid, +CacheAllocator::getNextCandidate(TierId tid, + PoolId pid, ClassId cid, unsigned int& searchTries) { typename NvmCacheT::PutToken token; Item* toRecycle = nullptr; Item* candidate = nullptr; - auto& mmContainer = getMMContainer(pid, cid); + auto& mmContainer = getMMContainer(tid, pid, cid); mmContainer.withEvictionIterator([this, pid, cid, &candidate, &toRecycle, &searchTries, &mmContainer, @@ -1351,13 +1408,13 @@ CacheAllocator::getNextCandidate(PoolId pid, template typename CacheAllocator::Item* -CacheAllocator::findEviction(PoolId pid, ClassId cid) { +CacheAllocator::findEviction(TierId tid, PoolId pid, ClassId cid) { // Keep searching for a candidate until we were able to evict it // or until the search limit has been exhausted unsigned int searchTries = 0; while (config_.evictionSearchTries == 0 || config_.evictionSearchTries > searchTries) { - auto [candidate, toRecycle] = getNextCandidate(pid, cid, searchTries); + auto [candidate, toRecycle] = getNextCandidate(tid, pid, cid, searchTries); // Reached the end of the eviction queue but doulen't find a candidate, // start again. @@ -1638,21 +1695,57 @@ void CacheAllocator::invalidateNvm(Item& item) { } } +template +TierId +CacheAllocator::getTierId(const Item& item) const { + return getTierId(item.getMemory()); +} + +template +TierId +CacheAllocator::getTierId(const void* ptr) const { + for (TierId tid = 0; tid < getNumTiers(); tid++) { + if (allocator_[tid]->isMemoryInAllocator(ptr)) + return tid; + } + + throw std::invalid_argument("Item does not belong to any tier!"); +} + template typename CacheAllocator::MMContainer& CacheAllocator::getMMContainer(const Item& item) const noexcept { + const auto tid = getTierId(item); const auto allocInfo = - allocator_->getAllocInfo(static_cast(&item)); - return getMMContainer(allocInfo.poolId, allocInfo.classId); + allocator_[tid]->getAllocInfo(static_cast(&item)); + return getMMContainer(tid, allocInfo.poolId, allocInfo.classId); } template typename CacheAllocator::MMContainer& -CacheAllocator::getMMContainer(PoolId pid, +CacheAllocator::getMMContainer(TierId tid, + PoolId pid, ClassId cid) const noexcept { - XDCHECK_LT(static_cast(pid), mmContainers_.size()); - XDCHECK_LT(static_cast(cid), mmContainers_[pid].size()); - return *mmContainers_[pid][cid]; + XDCHECK_LT(static_cast(tid), mmContainers_.size()); + XDCHECK_LT(static_cast(pid), mmContainers_[tid].size()); + XDCHECK_LT(static_cast(cid), mmContainers_[tid][pid].size()); + return *mmContainers_[tid][pid][cid]; +} + +template +MMContainerStat CacheAllocator::getMMContainerStat( + TierId tid, PoolId pid, ClassId cid) const noexcept { + if(static_cast(tid) >= mmContainers_.size()) { + return MMContainerStat{}; + } + if (static_cast(pid) >= mmContainers_[tid].size()) { + return MMContainerStat{}; + } + if (static_cast(cid) >= mmContainers_[tid][pid].size()) { + return MMContainerStat{}; + } + return mmContainers_[tid][pid][cid] ? mmContainers_[tid][pid][cid]->getStats() + : MMContainerStat{}; } template @@ -1846,8 +1939,9 @@ void CacheAllocator::markUseful(const ReadHandle& handle, template bool CacheAllocator::recordAccessInMMContainer(Item& item, AccessMode mode) { + const auto tid = getTierId(item); const auto allocInfo = - allocator_->getAllocInfo(static_cast(&item)); + allocator_[tid]->getAllocInfo(static_cast(&item)); (*stats_.cacheHits)[allocInfo.poolId][allocInfo.classId].inc(); // track recently accessed items if needed @@ -1855,14 +1949,15 @@ bool CacheAllocator::recordAccessInMMContainer(Item& item, ring_->trackItem(reinterpret_cast(&item), item.getSize()); } - auto& mmContainer = getMMContainer(allocInfo.poolId, allocInfo.classId); + auto& mmContainer = getMMContainer(tid, allocInfo.poolId, allocInfo.classId); return mmContainer.recordAccess(item, mode); } template uint32_t CacheAllocator::getUsableSize(const Item& item) const { + const auto tid = getTierId(item); const auto allocSize = - allocator_->getAllocInfo(static_cast(&item)).allocSize; + allocator_[tid]->getAllocInfo(static_cast(&item)).allocSize; return item.isChainedItem() ? allocSize - ChainedItem::getRequiredSize(0) : allocSize - Item::getRequiredSize(item.getKey(), 0); @@ -1871,8 +1966,10 @@ uint32_t CacheAllocator::getUsableSize(const Item& item) const { template typename CacheAllocator::SampleItem CacheAllocator::getSampleItem() { - size_t nvmCacheSize = nvmCache_ ? nvmCache_->getUsableSize() : 0; - size_t ramCacheSize = allocator_->getMemorySizeInclAdvised(); + // TODO: is using random tier a good idea? + auto tid = folly::Random::rand32() % getNumTiers(); + static size_t nvmCacheSize = nvmCache_ ? nvmCache_->getUsableSize() : 0; + static size_t ramCacheSize = allocator_[tid]->getMemorySizeInclAdvised(); bool fromNvm = folly::Random::rand64(0, nvmCacheSize + ramCacheSize) >= ramCacheSize; @@ -1881,19 +1978,18 @@ CacheAllocator::getSampleItem() { } // Sampling from DRAM cache - auto item = reinterpret_cast(allocator_->getRandomAlloc()); + auto item = reinterpret_cast(allocator_[tid]->getRandomAlloc()); if (!item) { return SampleItem{false /* fromNvm */}; } // Check that item returned is the same that was sampled - auto sharedHdl = std::make_shared(findInternal(item->getKey())); if (sharedHdl->get() != item) { return SampleItem{false /* fromNvm */}; } - const auto allocInfo = allocator_->getAllocInfo(item->getMemory()); + const auto allocInfo = allocator_[tid]->getAllocInfo(item->getMemory()); // Convert the Item to IOBuf to make SampleItem auto iobuf = folly::IOBuf{ @@ -1912,27 +2008,34 @@ CacheAllocator::getSampleItem() { template std::vector CacheAllocator::dumpEvictionIterator( - PoolId pid, ClassId cid, size_t numItems) { + PoolId pid, ClassId cid, size_t numItems) { if (numItems == 0) { return {}; } - if (static_cast(pid) >= mmContainers_.size() || - static_cast(cid) >= mmContainers_[pid].size()) { + // Always evict from the lowest layer. + int tid = getNumTiers() - 1; + + if (static_cast(tid) >= mmContainers_.size() || + static_cast(pid) >= mmContainers_[tid].size() || + static_cast(cid) >= mmContainers_[tid][pid].size()) { throw std::invalid_argument( - folly::sformat("Invalid PoolId: {} and ClassId: {}.", pid, cid)); + folly::sformat("Invalid TierId: {} and PoolId: {} and ClassId: {}.", tid, pid, cid)); } std::vector content; - auto& mm = *mmContainers_[pid][cid]; - - mm.withEvictionIterator([&content, numItems](auto&& itr) { - while (itr && content.size() < numItems) { - content.push_back(itr->toString()); - ++itr; - } - }); + size_t i = 0; + while (i < numItems && tid >= 0) { + auto& mm = *mmContainers_[tid][pid][cid]; + mm.withEvictionIterator([&content, numItems](auto&& itr) { + while (itr && content.size() < numItems) { + content.push_back(itr->toString()); + ++itr; + } + }); + --tid; + } return content; } @@ -2109,19 +2212,40 @@ PoolId CacheAllocator::addPool( std::shared_ptr resizeStrategy, bool ensureProvisionable) { folly::SharedMutex::WriteHolder w(poolsResizeAndRebalanceLock_); - auto pid = allocator_->addPool(name, size, allocSizes, ensureProvisionable); + + PoolId pid = 0; + size_t totalCacheSize = 0; + + for (TierId tid = 0; tid < getNumTiers(); tid++) { + totalCacheSize += allocator_[tid]->getMemorySize(); + } + + for (TierId tid = 0; tid < getNumTiers(); tid++) { + auto tierSizeRatio = + static_cast(allocator_[tid]->getMemorySize()) / totalCacheSize; + size_t tierPoolSize = static_cast(tierSizeRatio * size); + + // TODO: what if we manage to add pool only in one tier? + // we should probably remove that on failure + auto res = allocator_[tid]->addPool( + name, tierPoolSize, allocSizes, ensureProvisionable); + XDCHECK(tid == 0 || res == pid); + pid = res; + } + createMMContainers(pid, std::move(config)); setRebalanceStrategy(pid, std::move(rebalanceStrategy)); setResizeStrategy(pid, std::move(resizeStrategy)); + return pid; } template void CacheAllocator::overridePoolRebalanceStrategy( PoolId pid, std::shared_ptr rebalanceStrategy) { - if (static_cast(pid) >= mmContainers_.size()) { + if (static_cast(pid) >= mmContainers_[0].size()) { throw std::invalid_argument(folly::sformat( - "Invalid PoolId: {}, size of pools: {}", pid, mmContainers_.size())); + "Invalid PoolId: {}, size of pools: {}", pid, mmContainers_[0].size())); } setRebalanceStrategy(pid, std::move(rebalanceStrategy)); } @@ -2129,9 +2253,9 @@ void CacheAllocator::overridePoolRebalanceStrategy( template void CacheAllocator::overridePoolResizeStrategy( PoolId pid, std::shared_ptr resizeStrategy) { - if (static_cast(pid) >= mmContainers_.size()) { + if (static_cast(pid) >= mmContainers_[0].size()) { throw std::invalid_argument(folly::sformat( - "Invalid PoolId: {}, size of pools: {}", pid, mmContainers_.size())); + "Invalid PoolId: {}, size of pools: {}", pid, mmContainers_[0].size())); } setResizeStrategy(pid, std::move(resizeStrategy)); } @@ -2143,14 +2267,14 @@ void CacheAllocator::overridePoolOptimizeStrategy( } template -void CacheAllocator::overridePoolConfig(PoolId pid, +void CacheAllocator::overridePoolConfig(TierId tid, PoolId pid, const MMConfig& config) { - if (static_cast(pid) >= mmContainers_.size()) { + // TODO: add generic tier id checking + if (static_cast(pid) >= mmContainers_[tid].size()) { throw std::invalid_argument(folly::sformat( - "Invalid PoolId: {}, size of pools: {}", pid, mmContainers_.size())); + "Invalid PoolId: {}, size of pools: {}", pid, mmContainers_[tid].size())); } - - auto& pool = allocator_->getPool(pid); + auto& pool = allocator_[tid]->getPool(pid); for (unsigned int cid = 0; cid < pool.getNumClassId(); ++cid) { MMConfig mmConfig = config; mmConfig.addExtraConfig( @@ -2158,29 +2282,35 @@ void CacheAllocator::overridePoolConfig(PoolId pid, ? pool.getAllocationClass(static_cast(cid)) .getAllocsPerSlab() : 0); - DCHECK_NOTNULL(mmContainers_[pid][cid].get()); - mmContainers_[pid][cid]->setConfig(mmConfig); + DCHECK_NOTNULL(mmContainers_[tid][pid][cid].get()); + mmContainers_[tid][pid][cid]->setConfig(mmConfig); } } template void CacheAllocator::createMMContainers(const PoolId pid, MMConfig config) { - auto& pool = allocator_->getPool(pid); + // pools on each layer should have the same number of class id, etc. + // TODO: think about deduplication + auto& pool = allocator_[0]->getPool(pid); + for (unsigned int cid = 0; cid < pool.getNumClassId(); ++cid) { config.addExtraConfig( config_.trackTailHits ? pool.getAllocationClass(static_cast(cid)) .getAllocsPerSlab() : 0); - mmContainers_[pid][cid].reset(new MMContainer(config, compressor_)); + for (TierId tid = 0; tid < getNumTiers(); tid++) { + mmContainers_[tid][pid][cid].reset(new MMContainer(config, compressor_)); + } } } template PoolId CacheAllocator::getPoolId( folly::StringPiece name) const noexcept { - return allocator_->getPoolId(name.str()); + // each tier has the same pools + return allocator_[0]->getPoolId(name.str()); } // The Function returns a consolidated vector of Release Slab @@ -2223,7 +2353,9 @@ std::set CacheAllocator::filterCompactCachePools( template std::set CacheAllocator::getRegularPoolIds() const { folly::SharedMutex::ReadHolder r(poolsResizeAndRebalanceLock_); - return filterCompactCachePools(allocator_->getPoolIds()); + // TODO - get rid of the duplication - right now, each tier + // holds pool objects with mostly the same info + return filterCompactCachePools(allocator_[0]->getPoolIds()); } template @@ -2248,10 +2380,9 @@ std::set CacheAllocator::getRegularPoolIdsForResize() // getAdvisedMemorySize - then pools may be overLimit even when // all slabs are not allocated. Otherwise, pools may be overLimit // only after all slabs are allocated. - // - return (allocator_->allSlabsAllocated()) || - (allocator_->getAdvisedMemorySize() != 0) - ? filterCompactCachePools(allocator_->getPoolsOverLimit()) + return (allocator_[currentTier()]->allSlabsAllocated()) || + (allocator_[currentTier()]->getAdvisedMemorySize() != 0) + ? filterCompactCachePools(allocator_[currentTier()]->getPoolsOverLimit()) : std::set{}; } @@ -2262,7 +2393,7 @@ const std::string CacheAllocator::getCacheName() const { template PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { - const auto& pool = allocator_->getPool(poolId); + const auto& pool = allocator_[currentTier()]->getPool(poolId); const auto& allocSizes = pool.getAllocSizes(); auto mpStats = pool.getStats(); const auto& classIds = mpStats.classIds; @@ -2281,7 +2412,7 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { if (!isCompactCache) { for (const ClassId cid : classIds) { uint64_t classHits = (*stats_.cacheHits)[poolId][cid].get(); - XDCHECK(mmContainers_[poolId][cid], + XDCHECK(mmContainers_[currentTier()][poolId][cid], folly::sformat("Pid {}, Cid {} not initialized.", poolId, cid)); cacheStats.insert( {cid, @@ -2291,16 +2422,14 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { (*stats_.fragmentationSize)[poolId][cid].get(), classHits, (*stats_.chainedItemEvictions)[poolId][cid].get(), (*stats_.regularItemEvictions)[poolId][cid].get(), - mmContainers_[poolId][cid]->getStats()} - - }); + getMMContainerStat(currentTier(), poolId, cid)}}); totalHits += classHits; } } PoolStats ret; ret.isCompactCache = isCompactCache; - ret.poolName = allocator_->getPoolName(poolId); + ret.poolName = allocator_[currentTier()]->getPoolName(poolId); ret.poolSize = pool.getPoolSize(); ret.poolUsableSize = pool.getPoolUsableSize(); ret.poolAdvisedSize = pool.getPoolAdvisedSize(); @@ -2315,7 +2444,7 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { template ACStats CacheAllocator::getACStats(PoolId poolId, ClassId classId) const { - const auto& pool = allocator_->getPool(poolId); + const auto& pool = allocator_[currentTier()]->getPool(poolId); const auto& ac = pool.getAllocationClass(classId); return ac.getStats(); } @@ -2324,18 +2453,16 @@ template PoolEvictionAgeStats CacheAllocator::getPoolEvictionAgeStats( PoolId pid, unsigned int slabProjectionLength) const { PoolEvictionAgeStats stats; - - const auto& pool = allocator_->getPool(pid); + const auto& pool = allocator_[currentTier()]->getPool(pid); const auto& allocSizes = pool.getAllocSizes(); for (ClassId cid = 0; cid < static_cast(allocSizes.size()); ++cid) { - auto& mmContainer = getMMContainer(pid, cid); + auto& mmContainer = getMMContainer(currentTier(), pid, cid); const auto numItemsPerSlab = - allocator_->getPool(pid).getAllocationClass(cid).getAllocsPerSlab(); + allocator_[currentTier()]->getPool(pid).getAllocationClass(cid).getAllocsPerSlab(); const auto projectionLength = numItemsPerSlab * slabProjectionLength; stats.classEvictionAgeStats[cid] = mmContainer.getEvictionAgeStat(projectionLength); } - return stats; } @@ -2374,7 +2501,7 @@ void CacheAllocator::releaseSlab(PoolId pid, } try { - auto releaseContext = allocator_->startSlabRelease( + auto releaseContext = allocator_[currentTier()]->startSlabRelease( pid, victim, receiver, mode, hint, [this]() -> bool { return shutDownInProgress_; }); @@ -2383,15 +2510,15 @@ void CacheAllocator::releaseSlab(PoolId pid, return; } - releaseSlabImpl(releaseContext); - if (!allocator_->allAllocsFreed(releaseContext)) { + releaseSlabImpl(currentTier(), releaseContext); + if (!allocator_[currentTier()]->allAllocsFreed(releaseContext)) { throw std::runtime_error( folly::sformat("Was not able to free all allocs. PoolId: {}, AC: {}", releaseContext.getPoolId(), releaseContext.getClassId())); } - allocator_->completeSlabRelease(releaseContext); + allocator_[currentTier()]->completeSlabRelease(releaseContext); } catch (const exception::SlabReleaseAborted& e) { stats_.numAbortedSlabReleases.inc(); throw exception::SlabReleaseAborted(folly::sformat( @@ -2402,8 +2529,7 @@ void CacheAllocator::releaseSlab(PoolId pid, } template -SlabReleaseStats CacheAllocator::getSlabReleaseStats() - const noexcept { +SlabReleaseStats CacheAllocator::getSlabReleaseStats() const noexcept { std::lock_guard l(workersMutex_); return SlabReleaseStats{stats_.numActiveSlabReleases.get(), stats_.numReleasedForRebalance.get(), @@ -2421,7 +2547,7 @@ SlabReleaseStats CacheAllocator::getSlabReleaseStats() } template -void CacheAllocator::releaseSlabImpl( +void CacheAllocator::releaseSlabImpl(TierId tid, const SlabReleaseContext& releaseContext) { auto startTime = std::chrono::milliseconds(util::getCurrentTimeMs()); bool releaseStuck = false; @@ -2466,7 +2592,7 @@ void CacheAllocator::releaseSlabImpl( if (!isMoved) { evictForSlabRelease(releaseContext, item, throttler); } - XDCHECK(allocator_->isAllocFreed(releaseContext, alloc)); + XDCHECK(allocator_[tid]->isAllocFreed(releaseContext, alloc)); } } @@ -2545,10 +2671,11 @@ bool CacheAllocator::moveForSlabRelease( ctx.getPoolId(), ctx.getClassId()); }); } + auto tid = getTierId(oldItem); auto ref = oldItem.unmarkMoving(); XDCHECK_EQ(ref, 0); - const auto allocInfo = allocator_->getAllocInfo(oldItem.getMemory()); - allocator_->free(&oldItem); + const auto allocInfo = allocator_[tid]->getAllocInfo(oldItem.getMemory()); + allocator_[tid]->free(&oldItem); (*stats_.fragmentationSize)[allocInfo.poolId][allocInfo.classId].sub( util::getFragmentation(*this, oldItem)); @@ -2611,15 +2738,16 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { } const auto allocInfo = - allocator_->getAllocInfo(static_cast(&oldItem)); + allocator_[getTierId(oldItem)]->getAllocInfo(static_cast(&oldItem)); // Set up the destination for the move. Since oldItem would have the moving // bit set, it won't be picked for eviction. - auto newItemHdl = allocateInternal(allocInfo.poolId, - oldItem.getKey(), - oldItem.getSize(), - oldItem.getCreationTime(), - oldItem.getExpiryTime()); + auto newItemHdl = allocateInternalTier(getTierId(oldItem), + allocInfo.poolId, + oldItem.getKey(), + oldItem.getSize(), + oldItem.getCreationTime(), + oldItem.getExpiryTime()); if (!newItemHdl) { return {}; } @@ -2678,7 +2806,7 @@ void CacheAllocator::evictForSlabRelease( if (shutDownInProgress_) { item.unmarkMoving(); - allocator_->abortSlabRelease(ctx); + allocator_[getTierId(item)]->abortSlabRelease(ctx); throw exception::SlabReleaseAborted( folly::sformat("Slab Release aborted while trying to evict" " Item: {} Pool: {}, Class: {}.", @@ -2778,7 +2906,7 @@ void CacheAllocator::evictForSlabRelease( } const auto allocInfo = - allocator_->getAllocInfo(static_cast(evicted)); + allocator_[getTierId(item)]->getAllocInfo(static_cast(evicted)); if (evicted->hasChainedItem()) { (*stats_.chainedItemEvictions)[allocInfo.poolId][allocInfo.classId].inc(); } else { @@ -2829,18 +2957,20 @@ bool CacheAllocator::markMovingForSlabRelease( // At first, we assume this item was already freed bool itemFreed = true; bool markedMoving = false; - const auto fn = [&markedMoving, &itemFreed](void* memory) { + TierId tid = 0; + const auto fn = [&markedMoving, &itemFreed, &tid, this /* TODO - necessary for getTierId */](void* memory) { // Since this callback is executed, the item is not yet freed itemFreed = false; Item* item = static_cast(memory); if (item->markMoving()) { markedMoving = true; } + tid = getTierId(*item); }; auto startTime = util::getCurrentTimeSec(); while (true) { - allocator_->processAllocForRelease(ctx, alloc, fn); + allocator_[tid]->processAllocForRelease(ctx, alloc, fn); // If item is already freed we give up trying to mark the item moving // and return false, otherwise if marked as moving, we return true. @@ -2855,7 +2985,7 @@ bool CacheAllocator::markMovingForSlabRelease( itemFreed = true; if (shutDownInProgress_) { - allocator_->abortSlabRelease(ctx); + allocator_[tid]->abortSlabRelease(ctx); throw exception::SlabReleaseAborted( folly::sformat("Slab Release aborted while still trying to mark" " as moving for Item: {}. Pool: {}, Class: {}.", @@ -2878,12 +3008,15 @@ template CCacheT* CacheAllocator::addCompactCache(folly::StringPiece name, size_t size, Args&&... args) { + if (getNumTiers() != 1) + throw std::runtime_error("TODO: compact cache for multi-tier Cache not supported."); + if (!config_.isCompactCacheEnabled()) { throw std::logic_error("Compact cache is not enabled"); } folly::SharedMutex::WriteHolder lock(compactCachePoolsLock_); - auto poolId = allocator_->addPool(name, size, {Slab::kSize}); + auto poolId = allocator_[0]->addPool(name, size, {Slab::kSize}); isCompactCachePool_[poolId] = true; auto ptr = std::make_unique( @@ -2992,12 +3125,15 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { *metadata_.numChainedChildItems() = stats_.numChainedChildItems.get(); *metadata_.numAbortedSlabReleases() = stats_.numAbortedSlabReleases.get(); + // TODO: implement serialization for multiple tiers auto serializeMMContainers = [](MMContainers& mmContainers) { MMSerializationTypeContainer state; - for (unsigned int i = 0; i < mmContainers.size(); ++i) { + for (unsigned int i = 0; i < 1 /* TODO: */ ; ++i) { for (unsigned int j = 0; j < mmContainers[i].size(); ++j) { - if (mmContainers[i][j]) { - state.pools_ref()[i][j] = mmContainers[i][j]->saveState(); + for (unsigned int k = 0; k < mmContainers[i][j].size(); ++k) { + if (mmContainers[i][j][k]) { + state.pools_ref()[j][k] = mmContainers[i][j][k]->saveState(); + } } } } @@ -3007,7 +3143,8 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { serializeMMContainers(mmContainers_); AccessSerializationType accessContainerState = accessContainer_->saveState(); - MemoryAllocator::SerializationType allocatorState = allocator_->saveState(); + // TODO: foreach allocator + MemoryAllocator::SerializationType allocatorState = allocator_[0]->saveState(); CCacheManager::SerializationType ccState = compactCacheManager_->saveState(); AccessSerializationType chainedItemAccessContainerState = @@ -3069,6 +3206,8 @@ CacheAllocator::shutDown() { (shmShutDownStatus == ShmShutDownRes::kSuccess); shmManager_.reset(); + // TODO: save per-tier state + if (shmShutDownSucceeded) { if (!nvmShutDownStatusOpt || *nvmShutDownStatusOpt) return ShutDownStatus::kSuccess; @@ -3132,22 +3271,26 @@ CacheAllocator::deserializeMMContainers( const auto container = deserializer.deserialize(); - MMContainers mmContainers; + /* TODO: right now, we create empty containers because deserialization + * only works for a single (topmost) tier. */ + MMContainers mmContainers{getNumTiers()}; for (auto& kvPool : *container.pools_ref()) { auto i = static_cast(kvPool.first); auto& pool = getPool(i); for (auto& kv : kvPool.second) { auto j = static_cast(kv.first); - MMContainerPtr ptr = - std::make_unique(kv.second, - compressor); - auto config = ptr->getConfig(); - config.addExtraConfig(config_.trackTailHits - ? pool.getAllocationClass(j).getAllocsPerSlab() - : 0); - ptr->setConfig(config); - mmContainers[i][j] = std::move(ptr); + for (TierId tid = 0; tid < getNumTiers(); tid++) { + MMContainerPtr ptr = + std::make_unique(kv.second, + compressor); + auto config = ptr->getConfig(); + config.addExtraConfig(config_.trackTailHits + ? pool.getAllocationClass(j).getAllocsPerSlab() + : 0); + ptr->setConfig(config); + mmContainers[tid][i][j] = std::move(ptr); + } } } // We need to drop the unevictableMMContainer in the desierializer. @@ -3299,11 +3442,11 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { template CacheMemoryStats CacheAllocator::getCacheMemoryStats() const { - const auto totalCacheSize = allocator_->getMemorySize(); - const auto configuredTotalCacheSize = allocator_->getMemorySizeInclAdvised(); + const auto configuredTotalCacheSize = allocator_[currentTier()]->getMemorySizeInclAdvised(); + const auto totalCacheSize = allocator_[currentTier()]->getMemorySize(); auto addSize = [this](size_t a, PoolId pid) { - return a + allocator_->getPool(pid).getPoolSize(); + return a + allocator_[currentTier()]->getPool(pid).getPoolSize(); }; const auto regularPoolIds = getRegularPoolIds(); const auto ccCachePoolIds = getCCachePoolIds(); @@ -3316,9 +3459,9 @@ CacheMemoryStats CacheAllocator::getCacheMemoryStats() const { configuredTotalCacheSize, configuredRegularCacheSize, configuredCompactCacheSize, - allocator_->getAdvisedMemorySize(), + allocator_[currentTier()]->getAdvisedMemorySize(), memMonitor_ ? memMonitor_->getMaxAdvisePct() : 0, - allocator_->getUnreservedMemorySize(), + allocator_[currentTier()]->getUnreservedMemorySize(), nvmCache_ ? nvmCache_->getSize() : 0, util::getMemAvailable(), util::getRSSBytes()}; @@ -3511,6 +3654,8 @@ bool CacheAllocator::cleanupStrayShmSegments( // cache dir exists. clean up only if there are no other processes // attached. if another process was attached, the following would fail. ShmManager::cleanup(cacheDir, posix); + + // TODO: cleanup per-tier state } catch (const std::exception& e) { XLOGF(ERR, "Error cleaning up {}. Exception: ", cacheDir, e.what()); return false; @@ -3520,7 +3665,8 @@ bool CacheAllocator::cleanupStrayShmSegments( // Any other concurrent process can not be attached to the segments or // even if it does, we want to mark it for destruction. ShmManager::removeByName(cacheDir, detail::kShmInfoName, posix); - ShmManager::removeByName(cacheDir, detail::kShmCacheName, posix); + ShmManager::removeByName(cacheDir, detail::kShmCacheName + + std::to_string(0 /* TODO: per tier */), posix); ShmManager::removeByName(cacheDir, detail::kShmHashTableName, posix); ShmManager::removeByName(cacheDir, detail::kShmChainedItemHashTableName, posix); @@ -3534,14 +3680,16 @@ uint64_t CacheAllocator::getItemPtrAsOffset(const void* ptr) { // the two differ (e.g. Mac OS 12) - causing templating instantiation // errors downstream. + auto tid = getTierId(ptr); + // if this succeeeds, the address is valid within the cache. - allocator_->getAllocInfo(ptr); + allocator_[tid]->getAllocInfo(ptr); if (!isOnShm_ || !shmManager_) { throw std::invalid_argument("Shared memory not used"); } - const auto& shm = shmManager_->getShmByName(detail::kShmCacheName); + const auto& shm = shmManager_->getShmByName(detail::kShmCacheName + std::to_string(tid)); return reinterpret_cast(ptr) - reinterpret_cast(shm.getCurrentMapping().addr); diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 38c5cb7990..c3e50e09ed 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -806,7 +806,7 @@ class CacheAllocator : public CacheBase { // @param config new config for the pool // // @throw std::invalid_argument if the poolId is invalid - void overridePoolConfig(PoolId pid, const MMConfig& config); + void overridePoolConfig(TierId tid, PoolId pid, const MMConfig& config); // update an existing pool's rebalance strategy // @@ -847,8 +847,9 @@ class CacheAllocator : public CacheBase { // @return true if the operation succeeded. false if the size of the pool is // smaller than _bytes_ // @throw std::invalid_argument if the poolId is invalid. + // TODO: should call shrinkPool for specific tier? bool shrinkPool(PoolId pid, size_t bytes) { - return allocator_->shrinkPool(pid, bytes); + return allocator_[currentTier()]->shrinkPool(pid, bytes); } // grow an existing pool by _bytes_. This will fail if there is no @@ -857,8 +858,9 @@ class CacheAllocator : public CacheBase { // @return true if the pool was grown. false if the necessary number of // bytes were not available. // @throw std::invalid_argument if the poolId is invalid. + // TODO: should call growPool for specific tier? bool growPool(PoolId pid, size_t bytes) { - return allocator_->growPool(pid, bytes); + return allocator_[currentTier()]->growPool(pid, bytes); } // move bytes from one pool to another. The source pool should be at least @@ -871,7 +873,7 @@ class CacheAllocator : public CacheBase { // correct size to do the transfer. // @throw std::invalid_argument if src or dest is invalid pool bool resizePools(PoolId src, PoolId dest, size_t bytes) override { - return allocator_->resizePools(src, dest, bytes); + return allocator_[currentTier()]->resizePools(src, dest, bytes); } // Add a new compact cache with given name and size @@ -1076,12 +1078,13 @@ class CacheAllocator : public CacheBase { // @throw std::invalid_argument if the memory does not belong to this // cache allocator AllocInfo getAllocInfo(const void* memory) const { - return allocator_->getAllocInfo(memory); + return allocator_[getTierId(memory)]->getAllocInfo(memory); } // return the ids for the set of existing pools in this cache. std::set getPoolIds() const override final { - return allocator_->getPoolIds(); + // all tiers have the same pool ids. TODO: deduplicate + return allocator_[0]->getPoolIds(); } // return a list of pool ids that are backing compact caches. This includes @@ -1093,18 +1096,18 @@ class CacheAllocator : public CacheBase { // return the pool with speicified id. const MemoryPool& getPool(PoolId pid) const override final { - return allocator_->getPool(pid); + return allocator_[currentTier()]->getPool(pid); } // calculate the number of slabs to be advised/reclaimed in each pool PoolAdviseReclaimData calcNumSlabsToAdviseReclaim() override final { auto regularPoolIds = getRegularPoolIds(); - return allocator_->calcNumSlabsToAdviseReclaim(regularPoolIds); + return allocator_[currentTier()]->calcNumSlabsToAdviseReclaim(regularPoolIds); } // update number of slabs to advise in the cache void updateNumSlabsToAdvise(int32_t numSlabsToAdvise) override final { - allocator_->updateNumSlabsToAdvise(numSlabsToAdvise); + allocator_[currentTier()]->updateNumSlabsToAdvise(numSlabsToAdvise); } // returns a valid PoolId corresponding to the name or kInvalidPoolId if the @@ -1112,8 +1115,9 @@ class CacheAllocator : public CacheBase { PoolId getPoolId(folly::StringPiece name) const noexcept; // returns the pool's name by its poolId. - std::string getPoolName(PoolId poolId) const override { - return allocator_->getPoolName(poolId); + std::string getPoolName(PoolId poolId) const { + // all tiers have the same pool names. + return allocator_[0]->getPoolName(poolId); } // get stats related to all kinds of slab release events. @@ -1391,11 +1395,14 @@ class CacheAllocator : public CacheBase { using MMContainerPtr = std::unique_ptr; using MMContainers = - std::array, - MemoryPoolManager::kMaxPools>; + std::vector, + MemoryPoolManager::kMaxPools>>; void createMMContainers(const PoolId pid, MMConfig config); + TierId getTierId(const Item& item) const; + TierId getTierId(const void* ptr) const; + // acquire the MMContainer corresponding to the the Item's class and pool. // // @return pointer to the MMContainer. @@ -1403,7 +1410,12 @@ class CacheAllocator : public CacheBase { // allocation from the memory allocator. MMContainer& getMMContainer(const Item& item) const noexcept; - MMContainer& getMMContainer(PoolId pid, ClassId cid) const noexcept; + MMContainer& getMMContainer(TierId tid, PoolId pid, ClassId cid) const noexcept; + + // Get stats of the specified pid and cid. + // If such mmcontainer is not valid (pool id or cid out of bound) + // or the mmcontainer is not initialized, return an empty stat. + MMContainerStat getMMContainerStat(TierId tid, PoolId pid, ClassId cid) const noexcept; // create a new cache allocation. The allocation can be initialized // appropriately and made accessible through insert or insertOrReplace. @@ -1435,6 +1447,17 @@ class CacheAllocator : public CacheBase { uint32_t creationTime, uint32_t expiryTime); + // create a new cache allocation on specific memory tier. + // For description see allocateInternal. + // + // @param tid id a memory tier + WriteHandle allocateInternalTier(TierId tid, + PoolId id, + Key key, + uint32_t size, + uint32_t creationTime, + uint32_t expiryTime); + // Allocate a chained item // // The resulting chained item does not have a parent item and @@ -1525,6 +1548,15 @@ class CacheAllocator : public CacheBase { // not exist. FOLLY_ALWAYS_INLINE WriteHandle findFastImpl(Key key, AccessMode mode); + // Moves a regular item to a different memory tier. + // + // @param oldItem Reference to the item being moved + // @param newItemHdl Reference to the handle of the new item being moved into + // + // @return true If the move was completed, and the containers were updated + // successfully. + bool moveRegularItemOnEviction(Item& oldItem, WriteHandle& newItemHdl); + // Moves a regular item to a different slab. This should only be used during // slab release after the item's exclusive bit has been set. The user supplied // callback is responsible for copying the contents and fixing the semantics @@ -1679,15 +1711,17 @@ class CacheAllocator : public CacheBase { // Implementation to find a suitable eviction from the container. The // two parameters together identify a single container. // + // @param tid the id of the tier to look for evictions inside // @param pid the id of the pool to look for evictions inside // @param cid the id of the class to look for evictions inside // @return An evicted item or nullptr if there is no suitable candidate found // within the configured number of attempts. - Item* findEviction(PoolId pid, ClassId cid); + Item* findEviction(TierId tid, PoolId pid, ClassId cid); // Get next eviction candidate from MMContainer, remove from AccessContainer, // MMContainer and insert into NVMCache if enabled. // + // @param tid the id of the tier to look for evictions inside // @param pid the id of the pool to look for evictions inside // @param cid the id of the class to look for evictions inside // @param searchTries number of search attempts so far. @@ -1695,7 +1729,8 @@ class CacheAllocator : public CacheBase { // @return pair of [candidate, toRecycle]. Pair of null if reached the end of // the eviction queue or no suitable candidate found // within the configured number of attempts - std::pair getNextCandidate(PoolId pid, + std::pair getNextCandidate(TierId tid, + PoolId pid, ClassId cid, unsigned int& searchTries); @@ -1713,7 +1748,7 @@ class CacheAllocator : public CacheBase { const typename Item::PtrCompressor& compressor); unsigned int reclaimSlabs(PoolId id, size_t numSlabs) final { - return allocator_->reclaimSlabsAndGrow(id, numSlabs); + return allocator_[currentTier()]->reclaimSlabsAndGrow(id, numSlabs); } FOLLY_ALWAYS_INLINE EventTracker* getEventTracker() const { @@ -1772,7 +1807,7 @@ class CacheAllocator : public CacheBase { const void* hint = nullptr) final; // @param releaseContext slab release context - void releaseSlabImpl(const SlabReleaseContext& releaseContext); + void releaseSlabImpl(TierId tid, const SlabReleaseContext& releaseContext); // @return true when successfully marked as moving, // fasle when this item has already been freed @@ -1834,7 +1869,7 @@ class CacheAllocator : public CacheBase { // primitives. So we consciously exempt ourselves here from TSAN data race // detection. folly::annotate_ignore_thread_sanitizer_guard g(__FILE__, __LINE__); - auto slabsSkipped = allocator_->forEachAllocation(std::forward(f)); + auto slabsSkipped = allocator_[currentTier()]->forEachAllocation(std::forward(f)); stats().numReaperSkippedSlabs.add(slabsSkipped); } @@ -1878,10 +1913,10 @@ class CacheAllocator : public CacheBase { std::unique_ptr& worker, std::chrono::seconds timeout = std::chrono::seconds{0}); - ShmSegmentOpts createShmCacheOpts(); - std::unique_ptr createNewMemoryAllocator(); - std::unique_ptr restoreMemoryAllocator(); - std::unique_ptr restoreCCacheManager(); + ShmSegmentOpts createShmCacheOpts(TierId tid); + std::unique_ptr createNewMemoryAllocator(TierId tid); + std::unique_ptr restoreMemoryAllocator(TierId tid); + std::unique_ptr restoreCCacheManager(TierId tid); PoolIds filterCompactCachePools(const PoolIds& poolIds) const; @@ -1901,7 +1936,7 @@ class CacheAllocator : public CacheBase { } typename Item::PtrCompressor createPtrCompressor() const { - return allocator_->createPtrCompressor(); + return allocator_[0 /* TODO */]->createPtrCompressor(); } // helper utility to throttle and optionally log. @@ -1924,9 +1959,14 @@ class CacheAllocator : public CacheBase { // @param type the type of initialization // @return nullptr if the type is invalid - // @return pointer to memory allocator + // @return vector of pointers to memory allocator // @throw std::runtime_error if type is invalid - std::unique_ptr initAllocator(InitMemType type); + std::vector> initAllocator(InitMemType type); + + std::vector> createPrivateAllocator(); + std::vector> createAllocators(); + std::vector> restoreAllocators(); + // @param type the type of initialization // @return nullptr if the type is invalid // @return pointer to access container @@ -1984,6 +2024,17 @@ class CacheAllocator : public CacheBase { // BEGIN private members + TierId currentTier() const { + // TODO: every function which calls this method should be refactored. + // We should go case by case and either make such function work on + // all tiers or expose separate parameter to describe the tier ID. + return 0; + } + + unsigned getNumTiers() const { + return config_.memoryTierConfigs.size(); + } + // Whether the memory allocator for this cache allocator was created on shared // memory. The hash table, chained item hash table etc is also created on // shared memory except for temporary shared memory mode when they're created @@ -2009,9 +2060,10 @@ class CacheAllocator : public CacheBase { const MMConfig mmConfig_{}; // the memory allocator for allocating out of the available memory. - std::unique_ptr allocator_; + std::vector> allocator_; // compact cache allocator manager + // TODO: per tier? std::unique_ptr compactCacheManager_; // compact cache instances reside here when user "add" or "attach" compact diff --git a/cachelib/allocator/PoolOptimizer.cpp b/cachelib/allocator/PoolOptimizer.cpp index 8d67762be8..d101231a04 100644 --- a/cachelib/allocator/PoolOptimizer.cpp +++ b/cachelib/allocator/PoolOptimizer.cpp @@ -51,6 +51,8 @@ void PoolOptimizer::optimizeRegularPoolSizes() { void PoolOptimizer::optimizeCompactCacheSizes() { try { + // TODO: should optimizer look at each tier individually? + // If yes, then resizePools should be per-tier auto strategy = cache_.getPoolOptimizeStrategy(); if (!strategy) { strategy = strategy_; diff --git a/cachelib/allocator/memory/MemoryAllocator.h b/cachelib/allocator/memory/MemoryAllocator.h index 1ce58857de..625171fd6f 100644 --- a/cachelib/allocator/memory/MemoryAllocator.h +++ b/cachelib/allocator/memory/MemoryAllocator.h @@ -646,6 +646,13 @@ class MemoryAllocator { memoryPoolManager_.updateNumSlabsToAdvise(numSlabs); } + // returns ture if ptr points to memory which is managed by this + // allocator + bool isMemoryInAllocator(const void *ptr) { + return ptr && ptr >= slabAllocator_.getSlabMemoryBegin() + && ptr < slabAllocator_.getSlabMemoryEnd(); + } + private: // @param memory pointer to the memory. // @return the MemoryPool corresponding to the memory. diff --git a/cachelib/allocator/memory/SlabAllocator.h b/cachelib/allocator/memory/SlabAllocator.h index d82cf5b947..9fdb1e60b4 100644 --- a/cachelib/allocator/memory/SlabAllocator.h +++ b/cachelib/allocator/memory/SlabAllocator.h @@ -322,6 +322,17 @@ class SlabAllocator { return PtrCompressor(*this); } + // returns starting address of memory we own. + const Slab* getSlabMemoryBegin() const noexcept { + return reinterpret_cast(memoryStart_); + } + + // returns first byte after the end of memory region we own. + const Slab* getSlabMemoryEnd() const noexcept { + return reinterpret_cast(reinterpret_cast(memoryStart_) + + memorySize_); + } + private: // null Slab* presenttation. With 4M Slab size, a valid slab index would never // reach 2^16 - 1; @@ -339,12 +350,6 @@ class SlabAllocator { // @throw std::invalid_argument if the state is invalid. void checkState() const; - // returns first byte after the end of memory region we own. - const Slab* getSlabMemoryEnd() const noexcept { - return reinterpret_cast(reinterpret_cast(memoryStart_) + - memorySize_); - } - // returns true if we have slabbed all the memory that is available to us. // false otherwise. bool allMemorySlabbed() const noexcept { diff --git a/cachelib/allocator/tests/AllocatorResizeTest.h b/cachelib/allocator/tests/AllocatorResizeTest.h index d65205ac74..883dd9c056 100644 --- a/cachelib/allocator/tests/AllocatorResizeTest.h +++ b/cachelib/allocator/tests/AllocatorResizeTest.h @@ -966,23 +966,23 @@ class AllocatorResizeTest : public AllocatorTest { for (i = 1; i <= numItersToMaxAdviseAway + 1; i++) { alloc.memMonitor_->adviseAwaySlabs(); std::this_thread::sleep_for(std::chrono::seconds{2}); - ASSERT_EQ(alloc.allocator_->getAdvisedMemorySize(), i * perIterAdvSize); + ASSERT_EQ(alloc.allocator_[0 /* TODO - extend test */]->getAdvisedMemorySize(), i * perIterAdvSize); } i--; // This should fail alloc.memMonitor_->adviseAwaySlabs(); std::this_thread::sleep_for(std::chrono::seconds{2}); - auto totalAdvisedAwayMemory = alloc.allocator_->getAdvisedMemorySize(); + auto totalAdvisedAwayMemory = alloc.allocator_[0 /* TODO - extend test */]->getAdvisedMemorySize(); ASSERT_EQ(totalAdvisedAwayMemory, i * perIterAdvSize); // Try to reclaim back for (i = 1; i <= numItersToMaxAdviseAway + 1; i++) { alloc.memMonitor_->reclaimSlabs(); std::this_thread::sleep_for(std::chrono::seconds{2}); - ASSERT_EQ(alloc.allocator_->getAdvisedMemorySize(), + ASSERT_EQ(alloc.allocator_[0 /* TODO - extend test */]->getAdvisedMemorySize(), totalAdvisedAwayMemory - i * perIterAdvSize); } - totalAdvisedAwayMemory = alloc.allocator_->getAdvisedMemorySize(); + totalAdvisedAwayMemory = alloc.allocator_[0 /* TODO - extend test */]->getAdvisedMemorySize(); ASSERT_EQ(totalAdvisedAwayMemory, 0); } } diff --git a/cachelib/allocator/tests/BaseAllocatorTest.h b/cachelib/allocator/tests/BaseAllocatorTest.h index 3441815974..820edb8478 100644 --- a/cachelib/allocator/tests/BaseAllocatorTest.h +++ b/cachelib/allocator/tests/BaseAllocatorTest.h @@ -4257,13 +4257,13 @@ class BaseAllocatorTest : public AllocatorTest { // Had a bug: D4799860 where we allocated the wrong size for chained item { const auto parentAllocInfo = - alloc.allocator_->getAllocInfo(itemHandle->getMemory()); + alloc.allocator_[0 /* TODO - extend test */]->getAllocInfo(itemHandle->getMemory()); const auto child1AllocInfo = - alloc.allocator_->getAllocInfo(chainedItemHandle->getMemory()); + alloc.allocator_[0 /* TODO - extend test */]->getAllocInfo(chainedItemHandle->getMemory()); const auto child2AllocInfo = - alloc.allocator_->getAllocInfo(chainedItemHandle2->getMemory()); + alloc.allocator_[0 /* TODO - extend test */]->getAllocInfo(chainedItemHandle2->getMemory()); const auto child3AllocInfo = - alloc.allocator_->getAllocInfo(chainedItemHandle3->getMemory()); + alloc.allocator_[0 /* TODO - extend test */]->getAllocInfo(chainedItemHandle3->getMemory()); const auto parentCid = parentAllocInfo.classId; const auto child1Cid = child1AllocInfo.classId; diff --git a/cachelib/allocator/tests/TestBase-inl.h b/cachelib/allocator/tests/TestBase-inl.h index bf7355c87d..4d45e981bc 100644 --- a/cachelib/allocator/tests/TestBase-inl.h +++ b/cachelib/allocator/tests/TestBase-inl.h @@ -312,7 +312,7 @@ void AllocatorTest::testShmIsRemoved( ASSERT_FALSE(AllocatorT::ShmManager::segmentExists( config.getCacheDir(), detail::kShmHashTableName, config.usePosixShm)); ASSERT_FALSE(AllocatorT::ShmManager::segmentExists( - config.getCacheDir(), detail::kShmCacheName, config.usePosixShm)); + config.getCacheDir(), detail::kShmCacheName + std::to_string(0), config.usePosixShm)); ASSERT_FALSE(AllocatorT::ShmManager::segmentExists( config.getCacheDir(), detail::kShmChainedItemHashTableName, config.usePosixShm)); @@ -326,7 +326,7 @@ void AllocatorTest::testShmIsNotRemoved( ASSERT_TRUE(AllocatorT::ShmManager::segmentExists( config.getCacheDir(), detail::kShmHashTableName, config.usePosixShm)); ASSERT_TRUE(AllocatorT::ShmManager::segmentExists( - config.getCacheDir(), detail::kShmCacheName, config.usePosixShm)); + config.getCacheDir(), detail::kShmCacheName + std::to_string(0), config.usePosixShm)); ASSERT_TRUE(AllocatorT::ShmManager::segmentExists( config.getCacheDir(), detail::kShmChainedItemHashTableName, config.usePosixShm)); From 3cc41bdebf19afc4bd1f88bb3f70af3b6349401c Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Tue, 17 Jan 2023 10:49:16 -0800 Subject: [PATCH 05/31] AC stats multi-tier --- cachelib/allocator/Cache.h | 2 +- cachelib/allocator/CacheAllocator-inl.h | 5 +++-- cachelib/allocator/CacheAllocator.h | 2 +- cachelib/allocator/tests/CacheBaseTest.cpp | 2 +- cachelib/cachebench/cache/Cache-inl.h | 7 ++++--- cachelib/cachebench/cache/Cache.h | 4 ++-- cachelib/cachebench/cache/CacheStats.h | 18 ++++++++++-------- 7 files changed, 22 insertions(+), 18 deletions(-) diff --git a/cachelib/allocator/Cache.h b/cachelib/allocator/Cache.h index 082db65f7a..fed0abce2f 100644 --- a/cachelib/allocator/Cache.h +++ b/cachelib/allocator/Cache.h @@ -106,7 +106,7 @@ class CacheBase { // // @param poolId the pool id // @param classId the class id - virtual ACStats getACStats(PoolId poolId, ClassId classId) const = 0; + virtual ACStats getACStats(TierId tid, PoolId poolId, ClassId classId) const = 0; // @param poolId the pool id virtual AllSlabReleaseEvents getAllSlabReleaseEvents(PoolId poolId) const = 0; diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 5dd8a85a72..25f1b850ac 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -2442,9 +2442,10 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { } template -ACStats CacheAllocator::getACStats(PoolId poolId, +ACStats CacheAllocator::getACStats(TierId tid, + PoolId poolId, ClassId classId) const { - const auto& pool = allocator_[currentTier()]->getPool(poolId); + const auto& pool = allocator_[tid]->getPool(poolId); const auto& ac = pool.getAllocationClass(classId); return ac.getStats(); } diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index c3e50e09ed..9d70bdab01 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1190,7 +1190,7 @@ class CacheAllocator : public CacheBase { CacheMemoryStats getCacheMemoryStats() const override final; // return stats for Allocation Class - ACStats getACStats(PoolId pid, ClassId cid) const override final; + ACStats getACStats(TierId tid, PoolId pid, ClassId cid) const override final; // return the nvm cache stats map util::StatsMap getNvmCacheStatsMap() const override final; diff --git a/cachelib/allocator/tests/CacheBaseTest.cpp b/cachelib/allocator/tests/CacheBaseTest.cpp index f249786743..e7778d6ccf 100644 --- a/cachelib/allocator/tests/CacheBaseTest.cpp +++ b/cachelib/allocator/tests/CacheBaseTest.cpp @@ -34,7 +34,7 @@ class CacheBaseTest : public CacheBase, public SlabAllocatorTestBase { bool isObjectCache() const override { return false; } const MemoryPool& getPool(PoolId) const override { return memoryPool_; } PoolStats getPoolStats(PoolId) const override { return PoolStats(); } - ACStats getACStats(PoolId, ClassId) const { return ACStats(); }; + ACStats getACStats(TierId, PoolId, ClassId) const { return ACStats(); }; AllSlabReleaseEvents getAllSlabReleaseEvents(PoolId) const override { return AllSlabReleaseEvents{}; } diff --git a/cachelib/cachebench/cache/Cache-inl.h b/cachelib/cachebench/cache/Cache-inl.h index ab2908558f..08c9d05e39 100644 --- a/cachelib/cachebench/cache/Cache-inl.h +++ b/cachelib/cachebench/cache/Cache-inl.h @@ -643,14 +643,15 @@ Stats Cache::getStats() const { aggregate += poolStats; } - std::map> allocationClassStats{}; + std::map>> allocationClassStats{}; for (size_t pid = 0; pid < pools_.size(); pid++) { PoolId poolId = static_cast(pid); auto poolStats = cache_->getPoolStats(poolId); auto cids = poolStats.getClassIds(); - for (auto [cid, stats] : poolStats.mpStats.acStats) { - allocationClassStats[poolId][cid] = stats; + for (TierId tid = 0; tid < cache_->getNumTiers(); tid++) { + for (auto cid : cids) + allocationClassStats[tid][pid][cid] = cache_->getACStats(tid, pid, cid); } } diff --git a/cachelib/cachebench/cache/Cache.h b/cachelib/cachebench/cache/Cache.h index 65c70c30c1..a85c1efb66 100644 --- a/cachelib/cachebench/cache/Cache.h +++ b/cachelib/cachebench/cache/Cache.h @@ -325,8 +325,8 @@ class Cache { // return the stats for the pool. PoolStats getPoolStats(PoolId pid) const { return cache_->getPoolStats(pid); } - ACStats getACStats(PoolId pid, ClassId cid) const { - return cache_->getACStats(pid, cid); + ACStats getACStats(TierId tid, PoolId pid, ClassId cid) const { + return cache_->getACStats(tid, pid, cid); } // return the total number of inconsistent operations detected since start. diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index daca4bc80c..194249cc5e 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -101,7 +101,7 @@ struct Stats { uint64_t invalidDestructorCount{0}; int64_t unDestructedItemCount{0}; - std::map> allocationClassStats; + std::map>> allocationClassStats; // populate the counters related to nvm usage. Cache implementation can decide // what to populate since not all of those are interesting when running @@ -155,24 +155,26 @@ struct Stats { }; auto foreachAC = [&](auto cb) { - for (auto& pidStat : allocationClassStats) { - for (auto& cidStat : pidStat.second) { - cb(pidStat.first, cidStat.first, cidStat.second); + for (auto& tidStat : allocationClassStats) { + for (auto& pidStat : tidStat.second) { + for (auto& cidStat : pidStat.second) { + cb(tidStat.first, pidStat.first, cidStat.first, cidStat.second); + } } } }; - foreachAC([&](auto pid, auto cid, auto stats) { + foreachAC([&](auto tid, auto pid, auto cid, auto stats) { auto [allocSizeSuffix, allocSize] = formatMemory(stats.allocSize); auto [memorySizeSuffix, memorySize] = formatMemory(stats.totalAllocatedSize()); - out << folly::sformat("pid{:2} cid{:4} {:8.2f}{} memorySize: {:8.2f}{}", - pid, cid, allocSize, allocSizeSuffix, memorySize, + out << folly::sformat("tid{:2} pid{:2} cid{:4} {:8.2f}{} memorySize: {:8.2f}{}", + tid, pid, cid, allocSize, allocSizeSuffix, memorySize, memorySizeSuffix) << std::endl; }); - foreachAC([&](auto pid, auto cid, auto stats) { + foreachAC([&](auto tid, auto pid, auto cid, auto stats) { auto [allocSizeSuffix, allocSize] = formatMemory(stats.allocSize); // If the pool is not full, extrapolate usageFraction for AC assuming it From bf4c244a00cfbb762c884dc8eba18ed26071c7a3 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 8 Feb 2023 08:30:48 -0800 Subject: [PATCH 06/31] This commit contains the additional memory tiers tests for different pool sizes. We also use getPoolSize(pid), to get total size from all pools across allocators. It also fixes the tiering sizes (pulls changes from what was issue75 rebased commit that did not make it into upstream commits). Rebased to use ramCacheSize. --- cachelib/allocator/CacheAllocator-inl.h | 36 ++++++-- cachelib/allocator/CacheAllocator.h | 2 + .../tests/AllocatorMemoryTiersTest.cpp | 6 +- .../tests/AllocatorMemoryTiersTest.h | 40 ++++++++- cachelib/allocator/tests/MemoryTiersTest.cpp | 84 ++++++++++++++++++- 5 files changed, 155 insertions(+), 13 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 25f1b850ac..ea0bebf1ce 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -119,6 +119,16 @@ ShmSegmentOpts CacheAllocator::createShmCacheOpts(TierId tid) { return opts; } +template +size_t CacheAllocator::memoryTierSize(TierId tid) const { + auto partitions = std::accumulate(memoryTierConfigs.begin(), memoryTierConfigs.end(), 0UL, + [](const size_t i, const MemoryTierCacheConfig& config){ + return i + config.getRatio(); + }); + + return memoryTierConfigs[tid].calculateTierSize(config_.getCacheSize(), partitions); +} + template std::vector> CacheAllocator::createPrivateAllocator() { @@ -140,14 +150,15 @@ CacheAllocator::createPrivateAllocator() { template std::unique_ptr CacheAllocator::createNewMemoryAllocator(TierId tid) { + size_t tierSize = memoryTierSize(tid); return std::make_unique( getAllocatorConfig(config_), shmManager_ ->createShm(detail::kShmCacheName + std::to_string(tid), - config_.getCacheSize(), config_.slabMemoryBaseAddr, + tierSize, config_.slabMemoryBaseAddr, createShmCacheOpts(tid)) .addr, - config_.getCacheSize()); + tierSize); } template @@ -158,7 +169,7 @@ CacheAllocator::restoreMemoryAllocator(TierId tid) { shmManager_ ->attachShm(detail::kShmCacheName + std::to_string(tid), config_.slabMemoryBaseAddr, createShmCacheOpts(tid)).addr, - config_.getCacheSize(), + memoryTierSize(tid), config_.disableFullCoredump); } @@ -2391,6 +2402,16 @@ const std::string CacheAllocator::getCacheName() const { return config_.cacheName; } +template +size_t CacheAllocator::getPoolSize(PoolId poolId) const { + size_t poolSize = 0; + for (auto& allocator: allocator_) { + const auto& pool = allocator->getPool(poolId); + poolSize += pool.getPoolSize(); + } + return poolSize; +} + template PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { const auto& pool = allocator_[currentTier()]->getPool(poolId); @@ -3443,9 +3464,12 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { template CacheMemoryStats CacheAllocator::getCacheMemoryStats() const { - const auto configuredTotalCacheSize = allocator_[currentTier()]->getMemorySizeInclAdvised(); - const auto totalCacheSize = allocator_[currentTier()]->getMemorySize(); - + size_t totalCacheSize = 0; + size_t configuredTotalCacheSize = 0; + for(auto& allocator: allocator_) { + totalCacheSize += allocator->getMemorySize(); + configuredTotalCacheSize += allocator->getMemorySizeInclAdvised(); + } auto addSize = [this](size_t a, PoolId pid) { return a + allocator_[currentTier()]->getPool(pid).getPoolSize(); }; diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 9d70bdab01..bf1a570e85 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -2035,6 +2035,8 @@ class CacheAllocator : public CacheBase { return config_.memoryTierConfigs.size(); } + size_t memoryTierSize(TierId tid) const; + // Whether the memory allocator for this cache allocator was created on shared // memory. The hash table, chained item hash table etc is also created on // shared memory except for temporary shared memory mode when they're created diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp index 3e4847251f..c56f640847 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp @@ -23,9 +23,9 @@ namespace tests { using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest; // TODO(MEMORY_TIER): add more tests with different eviction policies -TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid1) { - this->testMultiTiersValid1(); -} +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); } } // end of namespace tests } // end of namespace cachelib diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h index a0d1513990..2ecb2c14ca 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h @@ -27,7 +27,7 @@ namespace tests { template class AllocatorMemoryTiersTest : public AllocatorTest { public: - void testMultiTiersValid1() { + void testMultiTiersInvalid() { typename AllocatorT::Config config; config.setCacheSize(100 * Slab::kSize); ASSERT_NO_THROW(config.configureMemoryTiers( @@ -36,6 +36,44 @@ class AllocatorMemoryTiersTest : public AllocatorTest { MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind( std::string("0"))})); } + + void testMultiTiersValid() { + typename AllocatorT::Config config; + config.setCacheSize(100 * Slab::kSize); + config.enableCachePersistence("/tmp"); + ASSERT_NO_THROW(config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind( + std::string("0")), + MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind( + std::string("0"))})); + + auto alloc = std::make_unique(AllocatorT::SharedMemNew, config); + ASSERT(alloc != nullptr); + + auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().ramCacheSize); + auto handle = alloc->allocate(pool, "key", std::string("value").size()); + ASSERT(handle != nullptr); + ASSERT_NO_THROW(alloc->insertOrReplace(handle)); + } + + void testMultiTiersValidMixed() { + typename AllocatorT::Config config; + config.setCacheSize(100 * Slab::kSize); + config.enableCachePersistence("/tmp"); + ASSERT_NO_THROW(config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind( + std::string("0")), + MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind( + std::string("0"))})); + + auto alloc = std::make_unique(AllocatorT::SharedMemNew, config); + ASSERT(alloc != nullptr); + + auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().ramCacheSize); + auto handle = alloc->allocate(pool, "key", std::string("value").size()); + ASSERT(handle != nullptr); + ASSERT_NO_THROW(alloc->insertOrReplace(handle)); + } }; } // namespace tests } // namespace cachelib diff --git a/cachelib/allocator/tests/MemoryTiersTest.cpp b/cachelib/allocator/tests/MemoryTiersTest.cpp index ed35115c0c..f618dcbb06 100644 --- a/cachelib/allocator/tests/MemoryTiersTest.cpp +++ b/cachelib/allocator/tests/MemoryTiersTest.cpp @@ -109,7 +109,7 @@ class MemoryTiersTest : public AllocatorTest { void validatePoolSize(PoolId poolId, std::unique_ptr& allocator, size_t expectedSize) { - size_t actualSize = allocator->getPool(poolId).getPoolSize(); + size_t actualSize = allocator->getPoolSize(poolId); EXPECT_EQ(actualSize, expectedSize); } @@ -119,9 +119,9 @@ class MemoryTiersTest : public AllocatorTest { size_t numTiers = 2) { if (isSizeValid) { auto pool = alloc->addPool("validPoolSize", poolSize); - EXPECT_LE(alloc->getPool(pool).getPoolSize(), poolSize); + EXPECT_LE(alloc->getPoolSize(pool), poolSize); if (poolSize >= numTiers * Slab::kSize) - EXPECT_GE(alloc->getPool(pool).getPoolSize(), + EXPECT_GE(alloc->getPoolSize(pool), poolSize - numTiers * Slab::kSize); } else { EXPECT_THROW(alloc->addPool("invalidPoolSize", poolSize), @@ -172,6 +172,84 @@ TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigRatioNotSet) { TEST_F(LruMemoryTiersTest, TestInvalid2TierConfigSizesNeCacheSize) { EXPECT_THROW(createTestCacheConfig({0, 0}), std::invalid_argument); } + +TEST_F(LruMemoryTiersTest, TestPoolAllocations) { + std::vector totalCacheSizes = {8 * GB, 2 * GB}; + + static const size_t numExtraSizes = 4; + static const size_t numExtraSlabs = 20; + + for (size_t i = 0; i < numExtraSizes; i++) { + totalCacheSizes.push_back(totalCacheSizes.back() + + (folly::Random::rand64() % numExtraSlabs) * + Slab::kSize); + } + + size_t min_ratio = 1; + size_t max_ratio = 111; + + static const size_t numCombinations = 10; + + for (auto totalCacheSize : totalCacheSizes) { + for (size_t k = 0; k < numCombinations; k++) { + const size_t i = folly::Random::rand32() % max_ratio + min_ratio; + const size_t j = folly::Random::rand32() % max_ratio + min_ratio; + LruAllocatorConfig cfg = + createTestCacheConfig({i, j}, + /* usePoisx */ true, totalCacheSize); + basicCheck(cfg, totalCacheSize); + + std::unique_ptr alloc = std::unique_ptr( + new LruAllocator(LruAllocator::SharedMemNew, cfg)); + + size_t size = (folly::Random::rand64() % + (alloc->getCacheMemoryStats().ramCacheSize - Slab::kSize)) + + Slab::kSize; + testAddPool(alloc, size, true); + } + } +} + +TEST_F(LruMemoryTiersTest, TestPoolInvalidAllocations) { + std::vector totalCacheSizes = {48 * MB, 51 * MB, 256 * MB, + 1 * GB, 5 * GB, 8 * GB}; + size_t min_ratio = 1; + size_t max_ratio = 111; + + static const size_t numCombinations = 10; + + for (auto totalCacheSize : totalCacheSizes) { + for (size_t k = 0; k < numCombinations; k++) { + const size_t i = folly::Random::rand32() % max_ratio + min_ratio; + const size_t j = folly::Random::rand32() % max_ratio + min_ratio; + LruAllocatorConfig cfg = + createTestCacheConfig({i, j}, + /* usePoisx */ true, totalCacheSize); + + std::unique_ptr alloc = nullptr; + try { + alloc = std::unique_ptr( + new LruAllocator(LruAllocator::SharedMemNew, cfg)); + } catch(...) { + // expection only if cache too small + size_t sum_ratios = std::accumulate( + cfg.getMemoryTierConfigs().begin(), cfg.getMemoryTierConfigs().end(), 0UL, + [](const size_t i, const MemoryTierCacheConfig& config) { + return i + config.getRatio(); + }); + auto tier1slabs = cfg.getMemoryTierConfigs()[0].calculateTierSize(cfg.getCacheSize(), sum_ratios) / Slab::kSize; + auto tier2slabs = cfg.getMemoryTierConfigs()[1].calculateTierSize(cfg.getCacheSize(), sum_ratios) / Slab::kSize; + EXPECT_TRUE(tier1slabs <= 2 || tier2slabs <= 2); + + continue; + } + + size_t size = (folly::Random::rand64() % (100 * GB)) + + alloc->getCacheMemoryStats().ramCacheSize; + testAddPool(alloc, size, false); + } + } +} } // namespace tests } // namespace cachelib } // namespace facebook From c432df6579c3c2b2a62949ed3b978d3a1ab7ee03 Mon Sep 17 00:00:00 2001 From: Sounak Gupta Date: Mon, 14 Nov 2022 02:07:57 -0800 Subject: [PATCH 07/31] This is the additional multi-tier support needed for the compressed ptr changes that were introduced upstream. - Includes later cosmetic changes added by sounak 9cb5c29fa493499192900227169050773820d265 --- cachelib/allocator/CacheAllocator.h | 3 +- cachelib/allocator/memory/AllocationClass.cpp | 11 ++-- cachelib/allocator/memory/AllocationClass.h | 2 +- cachelib/allocator/memory/CompressedPtr.h | 65 +++++++++++++++++-- cachelib/allocator/memory/MemoryAllocator.h | 11 ++-- cachelib/allocator/memory/SlabAllocator.h | 4 +- run_tests.sh | 1 + 7 files changed, 77 insertions(+), 20 deletions(-) diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index bf1a570e85..145f1e3dca 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1301,6 +1301,7 @@ class CacheAllocator : public CacheBase { sizeof(typename RefcountWithFlags::Value) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(KAllocation)) == sizeof(Item), "vtable overhead"); + // Check for CompressedPtr single/multi tier support static_assert(32 == sizeof(Item), "item overhead is 32 bytes"); // make sure there is no overhead in ChainedItem on top of a regular Item @@ -1936,7 +1937,7 @@ class CacheAllocator : public CacheBase { } typename Item::PtrCompressor createPtrCompressor() const { - return allocator_[0 /* TODO */]->createPtrCompressor(); + return typename Item::PtrCompressor(allocator_); } // helper utility to throttle and optionally log. diff --git a/cachelib/allocator/memory/AllocationClass.cpp b/cachelib/allocator/memory/AllocationClass.cpp index 71089153e9..512df86bbe 100644 --- a/cachelib/allocator/memory/AllocationClass.cpp +++ b/cachelib/allocator/memory/AllocationClass.cpp @@ -50,7 +50,7 @@ AllocationClass::AllocationClass(ClassId classId, poolId_(poolId), allocationSize_(allocSize), slabAlloc_(s), - freedAllocations_{slabAlloc_.createPtrCompressor()} { + freedAllocations_{slabAlloc_.createSingleTierPtrCompressor()} { checkState(); } @@ -102,7 +102,7 @@ AllocationClass::AllocationClass( currSlab_(s.getSlabForIdx(*object.currSlabIdx())), slabAlloc_(s), freedAllocations_(*object.freedAllocationsObject(), - slabAlloc_.createPtrCompressor()), + slabAlloc_.createSingleTierPtrCompressor()), canAllocate_(*object.canAllocate()) { if (!slabAlloc_.isRestorable()) { throw std::logic_error("The allocation class cannot be restored."); @@ -356,9 +356,10 @@ std::pair> AllocationClass::pruneFreeAllocs( // allocated slab, release any freed allocations belonging to this slab. // Set the bit to true if the corresponding allocation is freed, false // otherwise. - FreeList freeAllocs{slabAlloc_.createPtrCompressor()}; - FreeList notInSlab{slabAlloc_.createPtrCompressor()}; - FreeList inSlab{slabAlloc_.createPtrCompressor()}; + FreeList freeAllocs{slabAlloc_.createSingleTierPtrCompressor()}; + FreeList notInSlab{slabAlloc_.createSingleTierPtrCompressor()}; + FreeList inSlab{slabAlloc_.createSingleTierPtrCompressor()}; + lock_->lock_combine([&]() { // Take the allocation class free list offline diff --git a/cachelib/allocator/memory/AllocationClass.h b/cachelib/allocator/memory/AllocationClass.h index d45a45c6cd..269887f207 100644 --- a/cachelib/allocator/memory/AllocationClass.h +++ b/cachelib/allocator/memory/AllocationClass.h @@ -445,7 +445,7 @@ class AllocationClass { struct CACHELIB_PACKED_ATTR FreeAlloc { using CompressedPtr = facebook::cachelib::CompressedPtr; using PtrCompressor = - facebook::cachelib::PtrCompressor; + facebook::cachelib::SingleTierPtrCompressor; SListHook hook_{}; }; diff --git a/cachelib/allocator/memory/CompressedPtr.h b/cachelib/allocator/memory/CompressedPtr.h index 029abd91b9..d664063ea3 100644 --- a/cachelib/allocator/memory/CompressedPtr.h +++ b/cachelib/allocator/memory/CompressedPtr.h @@ -27,9 +27,12 @@ namespace cachelib { class SlabAllocator; +template +class PtrCompressor; + // This CompressedPtr makes decompression fast by staying away from division and -// modulo arithmetic and doing those during the compression time. We most often -// decompress a CompressedPtr than compress a pointer while creating one. This +// modulo arithmetic and doing those during the compression time. We most often +// decompress a CompressedPtr than compress a pointer while creating one. This // is used for pointer compression by the memory allocator. // We compress pointers by storing the tier index, slab index and alloc index of @@ -173,12 +176,14 @@ class CACHELIB_PACKED_ATTR CompressedPtr { } friend SlabAllocator; + template + friend class PtrCompressor; }; template -class PtrCompressor { +class SingleTierPtrCompressor { public: - explicit PtrCompressor(const AllocatorT& allocator) noexcept + explicit SingleTierPtrCompressor(const AllocatorT& allocator) noexcept : allocator_(allocator) {} const CompressedPtr compress(const PtrType* uncompressed) const { @@ -190,11 +195,11 @@ class PtrCompressor { allocator_.unCompress(compressed, false /* isMultiTiered */)); } - bool operator==(const PtrCompressor& rhs) const noexcept { + bool operator==(const SingleTierPtrCompressor& rhs) const noexcept { return &allocator_ == &rhs.allocator_; } - bool operator!=(const PtrCompressor& rhs) const noexcept { + bool operator!=(const SingleTierPtrCompressor& rhs) const noexcept { return !(*this == rhs); } @@ -202,5 +207,53 @@ class PtrCompressor { // memory allocator that does the pointer compression. const AllocatorT& allocator_; }; + +template +class PtrCompressor { + public: + explicit PtrCompressor(const AllocatorContainer& allocators) noexcept + : allocators_(allocators) {} + + const CompressedPtr compress(const PtrType* uncompressed) const { + if (uncompressed == nullptr) + return CompressedPtr{}; + + TierId tid; + for (tid = 0; tid < allocators_.size(); tid++) { + if (allocators_[tid]->isMemoryInAllocator( + static_cast(uncompressed))) + break; + } + + bool isMultiTiered = allocators_.size() > 1; + auto cptr = allocators_[tid]->compress(uncompressed, isMultiTiered); + if (isMultiTiered) { // config has multiple tiers + cptr.setTierId(tid); + } + return cptr; + } + + PtrType* unCompress(const CompressedPtr compressed) const { + if (compressed.isNull()) { + return nullptr; + } + bool isMultiTiered = allocators_.size() > 1; + auto& allocator = *allocators_[compressed.getTierId(isMultiTiered)]; + return static_cast( + allocator.unCompress(compressed, isMultiTiered)); + } + + bool operator==(const PtrCompressor& rhs) const noexcept { + return &allocators_ == &rhs.allocators_; + } + + bool operator!=(const PtrCompressor& rhs) const noexcept { + return !(*this == rhs); + } + + private: + // memory allocator that does the pointer compression. + const AllocatorContainer& allocators_; +}; } // namespace cachelib } // namespace facebook diff --git a/cachelib/allocator/memory/MemoryAllocator.h b/cachelib/allocator/memory/MemoryAllocator.h index 625171fd6f..a77d23494c 100644 --- a/cachelib/allocator/memory/MemoryAllocator.h +++ b/cachelib/allocator/memory/MemoryAllocator.h @@ -516,12 +516,13 @@ class MemoryAllocator { using CompressedPtr = facebook::cachelib::CompressedPtr; template using PtrCompressor = - facebook::cachelib::PtrCompressor; - + facebook::cachelib::PtrCompressor>>; + template - PtrCompressor createPtrCompressor() { - return slabAllocator_.createPtrCompressor(); - } + using SingleTierPtrCompressor = + facebook::cachelib::PtrCompressor; // compress a given pointer to a valid allocation made out of this allocator // through an allocate() or nullptr. Calling this otherwise with invalid diff --git a/cachelib/allocator/memory/SlabAllocator.h b/cachelib/allocator/memory/SlabAllocator.h index 9fdb1e60b4..a80a54672c 100644 --- a/cachelib/allocator/memory/SlabAllocator.h +++ b/cachelib/allocator/memory/SlabAllocator.h @@ -318,8 +318,8 @@ class SlabAllocator { } template - PtrCompressor createPtrCompressor() const { - return PtrCompressor(*this); + SingleTierPtrCompressor createSingleTierPtrCompressor() const { + return SingleTierPtrCompressor(*this); } // returns starting address of memory we own. diff --git a/run_tests.sh b/run_tests.sh index 111e218333..e575dbc62a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -2,6 +2,7 @@ # Newline separated list of tests to ignore BLACKLIST="allocator-test-NavySetupTest +allocator-test-NvmCacheTests shm-test-test_page_size" if [ "$1" == "long" ]; then From 4cefc4434e3f78b1caff9bb5e425083db324f6be Mon Sep 17 00:00:00 2001 From: Sounak Gupta Date: Thu, 21 Jul 2022 02:01:04 -0700 Subject: [PATCH 08/31] added per pool class rolling average latency (upstream PR version) fix for rolling stats (on multi-tier to be followed by multi-tier rolling stats implementation in the following commit) --- cachelib/allocator/CacheAllocator-inl.h | 12 ++- cachelib/allocator/CacheStats.cpp | 4 +- cachelib/allocator/CacheStatsInternal.h | 8 ++ .../allocator/memory/MemoryAllocatorStats.h | 4 + cachelib/cachebench/cache/CacheStats.h | 6 +- cachelib/common/RollingStats.h | 90 +++++++++++++++++++ 6 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 cachelib/common/RollingStats.h diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index ea0bebf1ce..0b82910ea6 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -382,6 +382,8 @@ CacheAllocator::allocateInternalTier(TierId tid, // the allocation class in our memory allocator. const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize); + util::RollingLatencyTracker rollTracker{ + (*stats_.classAllocLatency)[pid][cid]}; // TODO: per-tier (*stats_.allocAttempts)[pid][cid].inc(); @@ -478,8 +480,9 @@ CacheAllocator::allocateChainedItemInternal( const auto pid = allocator_[tid]->getAllocInfo(parent->getMemory()).poolId; const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize); - // TODO: per-tier? Right now stats_ are not used in any public periodic - // worker + util::RollingLatencyTracker rollTracker{ + (*stats_.classAllocLatency)[pid][cid]}; + (*stats_.allocAttempts)[pid][cid].inc(); void* memory = allocator_[tid]->allocate(pid, requiredSize); @@ -2468,7 +2471,10 @@ ACStats CacheAllocator::getACStats(TierId tid, ClassId classId) const { const auto& pool = allocator_[tid]->getPool(poolId); const auto& ac = pool.getAllocationClass(classId); - return ac.getStats(); + + auto stats = ac.getStats(); + stats.allocLatencyNs = (*stats_.classAllocLatency)[poolId][classId]; + return stats; } template diff --git a/cachelib/allocator/CacheStats.cpp b/cachelib/allocator/CacheStats.cpp index 1c0440e94a..c87d944643 100644 --- a/cachelib/allocator/CacheStats.cpp +++ b/cachelib/allocator/CacheStats.cpp @@ -44,6 +44,8 @@ void Stats::init() { initToZero(*fragmentationSize); initToZero(*chainedItemEvictions); initToZero(*regularItemEvictions); + + classAllocLatency = std::make_unique(); } template @@ -51,7 +53,7 @@ struct SizeVerify {}; void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { #ifndef SKIP_SIZE_VERIFY - SizeVerify a = SizeVerify<16176>{}; + SizeVerify a = SizeVerify<16192>{}; std::ignore = a; #endif ret.numCacheGets = numCacheGets.get(); diff --git a/cachelib/allocator/CacheStatsInternal.h b/cachelib/allocator/CacheStatsInternal.h index b2a5f8c469..8f54cd6ecf 100644 --- a/cachelib/allocator/CacheStatsInternal.h +++ b/cachelib/allocator/CacheStatsInternal.h @@ -21,6 +21,7 @@ #include "cachelib/allocator/Cache.h" #include "cachelib/allocator/memory/MemoryAllocator.h" #include "cachelib/common/AtomicCounter.h" +#include "cachelib/common/RollingStats.h" namespace facebook { namespace cachelib { @@ -229,6 +230,13 @@ struct Stats { std::unique_ptr chainedItemEvictions{}; std::unique_ptr regularItemEvictions{}; + using PerPoolClassRollingStats = + std::array, + MemoryPoolManager::kMaxPools>; + + // rolling latency tracking for every alloc class in every pool + std::unique_ptr classAllocLatency{}; + // Eviction failures due to parent cannot be removed from access container AtomicCounter evictFailParentAC{0}; diff --git a/cachelib/allocator/memory/MemoryAllocatorStats.h b/cachelib/allocator/memory/MemoryAllocatorStats.h index 65d82e000d..acda9ee530 100644 --- a/cachelib/allocator/memory/MemoryAllocatorStats.h +++ b/cachelib/allocator/memory/MemoryAllocatorStats.h @@ -20,6 +20,7 @@ #include #include "cachelib/allocator/memory/Slab.h" +#include "cachelib/common/RollingStats.h" namespace facebook { namespace cachelib { @@ -47,6 +48,9 @@ struct ACStats { // true if the allocation class is full. bool full; + // Rolling allocation latency (in ns) + util::RollingStats allocLatencyNs; + constexpr unsigned long long totalSlabs() const noexcept { return freeSlabs + usedSlabs; } diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index 194249cc5e..1a843b069f 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -184,8 +184,10 @@ struct Stats { : stats.usageFraction(); out << folly::sformat( - "pid{:2} cid{:4} {:8.2f}{} usageFraction: {:4.2f}", pid, cid, - allocSize, allocSizeSuffix, acUsageFraction) + "tid{:2} pid{:2} cid{:4} {:8.2f}{} usageFraction: {:4.2f} " + "rollingAvgAllocLatency: {:8.2f}ns", + tid, pid, cid, allocSize, allocSizeSuffix, acUsageFraction, + stats.allocLatencyNs.estimate()) << std::endl; }); } diff --git a/cachelib/common/RollingStats.h b/cachelib/common/RollingStats.h new file mode 100644 index 0000000000..4d179681ad --- /dev/null +++ b/cachelib/common/RollingStats.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cachelib/common/Utils.h" + +namespace facebook { +namespace cachelib { +namespace util { + +class RollingStats { + public: + // track latency by taking the value of duration directly. + void trackValue(double value) { + // This is a highly unlikely scenario where + // cnt_ reaches numerical limits. Skip update + // of the rolling average anymore. + if (cnt_ == std::numeric_limits::max()) { + cnt_ = 0; + return; + } + auto ratio = static_cast(cnt_) / (cnt_ + 1); + avg_ *= ratio; + ++cnt_; + avg_ += value / cnt_; + } + + // Return the rolling average. + double estimate() { return avg_; } + + private: + double avg_{0}; + uint64_t cnt_{0}; +}; + +class RollingLatencyTracker { + public: + explicit RollingLatencyTracker(RollingStats& stats) + : stats_(&stats), begin_(std::chrono::steady_clock::now()) {} + RollingLatencyTracker() {} + ~RollingLatencyTracker() { + if (stats_) { + auto tp = std::chrono::steady_clock::now(); + auto diffNanos = + std::chrono::duration_cast(tp - begin_) + .count(); + stats_->trackValue(static_cast(diffNanos)); + } + } + + RollingLatencyTracker(const RollingLatencyTracker&) = delete; + RollingLatencyTracker& operator=(const RollingLatencyTracker&) = delete; + + RollingLatencyTracker(RollingLatencyTracker&& rhs) noexcept + : stats_(rhs.stats_), begin_(rhs.begin_) { + rhs.stats_ = nullptr; + } + + RollingLatencyTracker& operator=(RollingLatencyTracker&& rhs) noexcept { + if (this != &rhs) { + this->~RollingLatencyTracker(); + new (this) RollingLatencyTracker(std::move(rhs)); + } + return *this; + } + + private: + RollingStats* stats_{nullptr}; + std::chrono::time_point begin_; +}; +} // namespace util +} // namespace cachelib +} // namespace facebook From 1f62a6323c759886f6ea32d45edea8e59bc722a3 Mon Sep 17 00:00:00 2001 From: Sounak Gupta Date: Thu, 21 Jul 2022 02:01:04 -0700 Subject: [PATCH 09/31] added per tier pool class rolling average latency (based on upstream PR) --- cachelib/allocator/Cache.h | 3 +++ cachelib/allocator/CacheAllocator-inl.h | 10 ++++++---- cachelib/allocator/CacheStats.cpp | 2 +- cachelib/allocator/CacheStats.h | 1 + cachelib/allocator/CacheStatsInternal.h | 7 ++++--- cachelib/cachebench/cache/CacheStats.h | 11 +++-------- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cachelib/allocator/Cache.h b/cachelib/allocator/Cache.h index fed0abce2f..cb2fa83f0d 100644 --- a/cachelib/allocator/Cache.h +++ b/cachelib/allocator/Cache.h @@ -85,6 +85,9 @@ class CacheBase { CacheBase(CacheBase&&) = default; CacheBase& operator=(CacheBase&&) = default; + // TODO: come up with some reasonable number + static constexpr unsigned kMaxTiers = 2; + // Get a string referring to the cache name for this cache virtual const std::string getCacheName() const = 0; diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 0b82910ea6..b70dd028bf 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -383,7 +383,7 @@ CacheAllocator::allocateInternalTier(TierId tid, // the allocation class in our memory allocator. const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize); util::RollingLatencyTracker rollTracker{ - (*stats_.classAllocLatency)[pid][cid]}; + (*stats_.classAllocLatency)[tid][pid][cid]}; // TODO: per-tier (*stats_.allocAttempts)[pid][cid].inc(); @@ -481,8 +481,10 @@ CacheAllocator::allocateChainedItemInternal( const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize); util::RollingLatencyTracker rollTracker{ - (*stats_.classAllocLatency)[pid][cid]}; - + (*stats_.classAllocLatency)[tid][pid][cid]}; + + // TODO: per-tier? Right now stats_ are not used in any public periodic + // worker (*stats_.allocAttempts)[pid][cid].inc(); void* memory = allocator_[tid]->allocate(pid, requiredSize); @@ -2473,7 +2475,7 @@ ACStats CacheAllocator::getACStats(TierId tid, const auto& ac = pool.getAllocationClass(classId); auto stats = ac.getStats(); - stats.allocLatencyNs = (*stats_.classAllocLatency)[poolId][classId]; + stats.allocLatencyNs = (*stats_.classAllocLatency)[tid][poolId][classId]; return stats; } diff --git a/cachelib/allocator/CacheStats.cpp b/cachelib/allocator/CacheStats.cpp index c87d944643..55305c7903 100644 --- a/cachelib/allocator/CacheStats.cpp +++ b/cachelib/allocator/CacheStats.cpp @@ -45,7 +45,7 @@ void Stats::init() { initToZero(*chainedItemEvictions); initToZero(*regularItemEvictions); - classAllocLatency = std::make_unique(); + classAllocLatency = std::make_unique(); } template diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index 0bdfc5db5d..5de9ced173 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -25,6 +25,7 @@ #include "cachelib/allocator/memory/Slab.h" #include "cachelib/common/FastStats.h" #include "cachelib/common/PercentileStats.h" +#include "cachelib/common/RollingStats.h" #include "cachelib/common/Time.h" namespace facebook { diff --git a/cachelib/allocator/CacheStatsInternal.h b/cachelib/allocator/CacheStatsInternal.h index 8f54cd6ecf..19a15fbbd4 100644 --- a/cachelib/allocator/CacheStatsInternal.h +++ b/cachelib/allocator/CacheStatsInternal.h @@ -230,12 +230,13 @@ struct Stats { std::unique_ptr chainedItemEvictions{}; std::unique_ptr regularItemEvictions{}; - using PerPoolClassRollingStats = + using PerTierPoolClassRollingStats = std::array< std::array, - MemoryPoolManager::kMaxPools>; + MemoryPoolManager::kMaxPools>, + CacheBase::kMaxTiers>; // rolling latency tracking for every alloc class in every pool - std::unique_ptr classAllocLatency{}; + std::unique_ptr classAllocLatency{}; // Eviction failures due to parent cannot be removed from access container AtomicCounter evictFailParentAC{0}; diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index 1a843b069f..6d118f0d24 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -164,18 +164,11 @@ struct Stats { } }; + foreachAC([&](auto tid, auto pid, auto cid, auto stats) { auto [allocSizeSuffix, allocSize] = formatMemory(stats.allocSize); auto [memorySizeSuffix, memorySize] = formatMemory(stats.totalAllocatedSize()); - out << folly::sformat("tid{:2} pid{:2} cid{:4} {:8.2f}{} memorySize: {:8.2f}{}", - tid, pid, cid, allocSize, allocSizeSuffix, memorySize, - memorySizeSuffix) - << std::endl; - }); - - foreachAC([&](auto tid, auto pid, auto cid, auto stats) { - auto [allocSizeSuffix, allocSize] = formatMemory(stats.allocSize); // If the pool is not full, extrapolate usageFraction for AC assuming it // will grow at the same rate. This value will be the same for all ACs. @@ -185,8 +178,10 @@ struct Stats { out << folly::sformat( "tid{:2} pid{:2} cid{:4} {:8.2f}{} usageFraction: {:4.2f} " + "memorySize: {:8.2f}{} " "rollingAvgAllocLatency: {:8.2f}ns", tid, pid, cid, allocSize, allocSizeSuffix, acUsageFraction, + memorySize, memorySizeSuffix, stats.allocLatencyNs.estimate()) << std::endl; }); From 489ef201daa822fb168c0e3c0f42322cbc7afbb1 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Tue, 9 Aug 2022 10:45:26 -0400 Subject: [PATCH 10/31] MM2Q promotion iterators (#1) Hot queue iterator for 2Q. Will start at Hot queue and move to Warm queue if hot queue is exhausted. Useful for promotion semantics if using 2Q replacement. rebased on to develop and added some tests. --- cachelib/allocator/MM2Q-inl.h | 10 ++++ cachelib/allocator/MM2Q.h | 5 ++ cachelib/allocator/datastruct/DList.h | 4 ++ .../allocator/datastruct/MultiDList-inl.h | 56 ++++++++++++++++--- cachelib/allocator/datastruct/MultiDList.h | 16 +++++- cachelib/allocator/tests/MM2QTest.cpp | 33 +++++++++++ cachelib/allocator/tests/MMTypeTest.h | 2 + 7 files changed, 116 insertions(+), 10 deletions(-) diff --git a/cachelib/allocator/MM2Q-inl.h b/cachelib/allocator/MM2Q-inl.h index ba388d40a4..07aae775f7 100644 --- a/cachelib/allocator/MM2Q-inl.h +++ b/cachelib/allocator/MM2Q-inl.h @@ -258,6 +258,16 @@ void MM2Q::Container::withEvictionIterator(F&& fun) { } } +// returns the head of the hot queue for promotion +template T::*HookPtr> +template +void +MM2Q::Container::withPromotionIterator(F&& fun) { + lruMutex_->lock_combine([this, &fun]() { + fun(LockedIterator{LockHolder{}, lru_.begin(LruType::Hot)}); + }); +} + template T::*HookPtr> void MM2Q::Container::removeLocked(T& node, bool doRebalance) noexcept { diff --git a/cachelib/allocator/MM2Q.h b/cachelib/allocator/MM2Q.h index 9c5cac834c..62b644f9c6 100644 --- a/cachelib/allocator/MM2Q.h +++ b/cachelib/allocator/MM2Q.h @@ -502,6 +502,11 @@ class MM2Q { // Iterator passed as parameter. template void withEvictionIterator(F&& f); + + // Execute provided function under container lock. Function gets + // iterator passed as parameter. + template + void withPromotionIterator(F&& f); // get the current config as a copy Config getConfig() const; diff --git a/cachelib/allocator/datastruct/DList.h b/cachelib/allocator/datastruct/DList.h index 2e872c8ee0..4d862b1908 100644 --- a/cachelib/allocator/datastruct/DList.h +++ b/cachelib/allocator/datastruct/DList.h @@ -221,6 +221,10 @@ class DList { curr_ = dir_ == Direction::FROM_HEAD ? dlist_->head_ : dlist_->tail_; } + Direction getDirection() noexcept { + return dir_; + } + protected: void goForward() noexcept; void goBackward() noexcept; diff --git a/cachelib/allocator/datastruct/MultiDList-inl.h b/cachelib/allocator/datastruct/MultiDList-inl.h index e20510d4fc..cd79b600c5 100644 --- a/cachelib/allocator/datastruct/MultiDList-inl.h +++ b/cachelib/allocator/datastruct/MultiDList-inl.h @@ -25,12 +25,26 @@ void MultiDList::Iterator::goForward() noexcept { } // Move iterator forward ++currIter_; - // If we land at the rend of this list, move to the previous list. - while (index_ != kInvalidIndex && - currIter_ == mlist_.lists_[index_]->rend()) { - --index_; - if (index_ != kInvalidIndex) { - currIter_ = mlist_.lists_[index_]->rbegin(); + + if (currIter_.getDirection() == DListIterator::Direction::FROM_HEAD) { + // If we land at the rend of this list, move to the previous list. + while (index_ != kInvalidIndex && index_ != mlist_.lists_.size() && + currIter_ == mlist_.lists_[index_]->end()) { + ++index_; + if (index_ != kInvalidIndex && index_ != mlist_.lists_.size()) { + currIter_ = mlist_.lists_[index_]->begin(); + } else { + return; + } + } + } else { + // If we land at the rend of this list, move to the previous list. + while (index_ != kInvalidIndex && + currIter_ == mlist_.lists_[index_]->rend()) { + --index_; + if (index_ != kInvalidIndex) { + currIter_ = mlist_.lists_[index_]->rbegin(); + } } } } @@ -71,6 +85,25 @@ void MultiDList::Iterator::initToValidRBeginFrom( : mlist_.lists_[index_]->rbegin(); } +template T::*HookPtr> +void MultiDList::Iterator::initToValidBeginFrom( + size_t listIdx) noexcept { + // Find the first non-empty list. + index_ = listIdx; + while (index_ != mlist_.lists_.size() && + mlist_.lists_[index_]->size() == 0) { + ++index_; + } + if (index_ == mlist_.lists_.size()) { + //we reached the end - we should get set to + //invalid index + index_ = std::numeric_limits::max(); + } + currIter_ = index_ == std::numeric_limits::max() + ? mlist_.lists_[0]->begin() + : mlist_.lists_[index_]->begin(); +} + template T::*HookPtr> typename MultiDList::Iterator& MultiDList::Iterator::operator++() noexcept { @@ -97,7 +130,16 @@ typename MultiDList::Iterator MultiDList::rbegin( if (listIdx >= lists_.size()) { throw std::invalid_argument("Invalid list index for MultiDList iterator."); } - return MultiDList::Iterator(*this, listIdx); + return MultiDList::Iterator(*this, listIdx, false); +} + +template T::*HookPtr> +typename MultiDList::Iterator MultiDList::begin( + size_t listIdx) const { + if (listIdx >= lists_.size()) { + throw std::invalid_argument("Invalid list index for MultiDList iterator."); + } + return MultiDList::Iterator(*this, listIdx, true); } template T::*HookPtr> diff --git a/cachelib/allocator/datastruct/MultiDList.h b/cachelib/allocator/datastruct/MultiDList.h index 1a59baa715..bd7be00bd4 100644 --- a/cachelib/allocator/datastruct/MultiDList.h +++ b/cachelib/allocator/datastruct/MultiDList.h @@ -110,14 +110,18 @@ class MultiDList { } explicit Iterator(const MultiDList& mlist, - size_t listIdx) noexcept + size_t listIdx, bool head) noexcept : currIter_(mlist.lists_[mlist.lists_.size() - 1]->rbegin()), mlist_(mlist) { XDCHECK_LT(listIdx, mlist.lists_.size()); - initToValidRBeginFrom(listIdx); + if (head) { + initToValidBeginFrom(listIdx); + } else { + initToValidRBeginFrom(listIdx); + } // We should either point to an element or the end() iterator // which has an invalid index_. - XDCHECK(index_ == kInvalidIndex || currIter_.get() != nullptr); + XDCHECK(index_ == kInvalidIndex || index_ == mlist.lists_.size() || currIter_.get() != nullptr); } virtual ~Iterator() = default; @@ -169,6 +173,9 @@ class MultiDList { // reset iterator to the beginning of a speicific queue void initToValidRBeginFrom(size_t listIdx) noexcept; + + // reset iterator to the head of a specific queue + void initToValidBeginFrom(size_t listIdx) noexcept; // Index of current list size_t index_{0}; @@ -184,6 +191,9 @@ class MultiDList { // provides an iterator starting from the tail of a specific list. Iterator rbegin(size_t idx) const; + + // provides an iterator starting from the head of a specific list. + Iterator begin(size_t idx) const; // Iterator to compare against for the end. Iterator rend() const noexcept; diff --git a/cachelib/allocator/tests/MM2QTest.cpp b/cachelib/allocator/tests/MM2QTest.cpp index e11dd95f5a..0e01ffa56f 100644 --- a/cachelib/allocator/tests/MM2QTest.cpp +++ b/cachelib/allocator/tests/MM2QTest.cpp @@ -223,6 +223,19 @@ void MMTypeTest::testIterate(std::vector>& nodes, } } +template +void MMTypeTest::testIterateHot(std::vector>& nodes, + Container& c) { + auto it = nodes.rbegin(); + c.withPromotionIterator([&it,&c](auto &&it2q) { + while (it2q && c.isHot(*it2q)) { + ASSERT_EQ(it2q->getId(), (*it)->getId()); + ++it2q; + ++it; + } + }); +} + template void MMTypeTest::testMatch(std::string expected, MMTypeTest::Container& c) { @@ -238,6 +251,23 @@ void MMTypeTest::testMatch(std::string expected, ASSERT_EQ(expected, actual); } +template +void MMTypeTest::testMatchHot(std::string expected, + MMTypeTest::Container& c) { + int index = -1; + std::string actual; + c.withPromotionIterator([&c,&actual,&index](auto &&it2q) { + while (it2q) { + ++index; + actual += folly::stringPrintf( + "%d:%s, ", it2q->getId(), + (c.isHot(*it2q) ? "H" : (c.isCold(*it2q) ? "C" : "W"))); + ++it2q; + } + }); + ASSERT_EQ(expected, actual); +} + TEST_F(MM2QTest, DetailedTest) { MM2Q::Config config; config.lruRefreshTime = 0; @@ -259,8 +289,11 @@ TEST_F(MM2QTest, DetailedTest) { } testIterate(nodes, c); + testIterateHot(nodes, c); testMatch("0:C, 1:C, 2:C, 3:C, 4:H, 5:H, ", c); + testMatchHot("5:H, 4:H, 3:C, 2:C, 1:C, 0:C, ", c); + // Move 3 to top of the hot cache c.recordAccess(*(nodes[4]), AccessMode::kRead); testMatch("0:C, 1:C, 2:C, 3:C, 5:H, 4:H, ", c); diff --git a/cachelib/allocator/tests/MMTypeTest.h b/cachelib/allocator/tests/MMTypeTest.h index d38f6ce2c1..dbc55677ea 100644 --- a/cachelib/allocator/tests/MMTypeTest.h +++ b/cachelib/allocator/tests/MMTypeTest.h @@ -147,7 +147,9 @@ class MMTypeTest : public testing::Test { void testRecordAccessBasic(Config c); void testSerializationBasic(Config c); void testIterate(std::vector>& nodes, Container& c); + void testIterateHot(std::vector>& nodes, Container& c); void testMatch(std::string expected, Container& c); + void testMatchHot(std::string expected, Container& c); size_t getListSize(const Container& c, typename MMType::LruType list); void verifyIterationVariants(Container& c); }; From 048c809fa3438cdd74b2faae140c8fbf0ac8646a Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Mon, 6 Feb 2023 16:45:18 -0800 Subject: [PATCH 11/31] CS Patch Part 2 for mulit-tier cachelib: - transparent item movement - multi-tier combined locking with exclusive bit (#38) with refactored incRef to support returning the result of markMoving (fail if already moving or exclusvie bit is set) option. - add tests (updated for numa bindings - post combined locking) for transparent item movement --- cachelib/allocator/CacheAllocator-inl.h | 402 +++++++++++++++--- cachelib/allocator/CacheAllocator.h | 145 ++++++- cachelib/allocator/CacheItem-inl.h | 4 +- cachelib/allocator/CacheItem.h | 6 +- cachelib/allocator/Handle.h | 9 +- cachelib/allocator/MMLru-inl.h | 12 + cachelib/allocator/MMLru.h | 5 +- cachelib/allocator/Refcount.h | 57 ++- .../tests/AllocatorMemoryTiersTest.cpp | 2 + .../tests/AllocatorMemoryTiersTest.h | 94 ++++ cachelib/allocator/tests/ItemHandleTest.cpp | 8 + cachelib/allocator/tests/ItemTest.cpp | 4 +- cachelib/allocator/tests/RefCountTest.cpp | 22 +- cachelib/cachebench/util/CacheConfig.h | 2 +- 14 files changed, 670 insertions(+), 102 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index b70dd028bf..47d4871f6b 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -84,6 +84,8 @@ CacheAllocator::CacheAllocator( config.chainedItemAccessConfig)), chainedItemLocks_(config_.chainedItemsLockPower, std::make_shared()), + movesMap_(kShards), + moveLock_(kShards), cacheCreationTime_{ type != InitMemType::kMemAttach ? util::getCurrentTimeSec() @@ -121,6 +123,7 @@ ShmSegmentOpts CacheAllocator::createShmCacheOpts(TierId tid) { template size_t CacheAllocator::memoryTierSize(TierId tid) const { + auto& memoryTierConfigs = config_.memoryTierConfigs; auto partitions = std::accumulate(memoryTierConfigs.begin(), memoryTierConfigs.end(), 0UL, [](const size_t i, const MemoryTierCacheConfig& config){ return i + config.getRatio(); @@ -540,14 +543,15 @@ void CacheAllocator::addChainedItem(WriteHandle& parent, // Count a new child stats_.numChainedChildItems.inc(); - insertInMMContainer(*child); - // Increment refcount since this chained item is now owned by the parent // Parent will decrement the refcount upon release. Since this is an // internal refcount, we dont include it in active handle tracking. - child->incRef(); + auto ret = child->incRef(true); + XDCHECK(ret == RefcountWithFlags::incResult::incOk); XDCHECK_EQ(2u, child->getRefCount()); + insertInMMContainer(*child); + invalidateNvm(*parent); if (auto eventTracker = getEventTracker()) { eventTracker->record(AllocatorApiEvent::ADD_CHAINED, parent->getKey(), @@ -791,7 +795,8 @@ CacheAllocator::replaceChainedItemLocked(Item& oldItem, // Since this is an internal refcount, we dont include it in active handle // tracking. - newItemHdl->incRef(); + auto ret = newItemHdl->incRef(true); + XDCHECK(ret == RefcountWithFlags::incResult::incOk); return oldItemHdl; } @@ -957,12 +962,12 @@ CacheAllocator::releaseBackToAllocator(Item& it, } template -bool CacheAllocator::incRef(Item& it) { - if (it.incRef()) { - ++handleCount_.tlStats(); - return true; - } - return false; +RefcountWithFlags::incResult CacheAllocator::incRef(Item& it, bool failIfMoving) { + auto ret = it.incRef(failIfMoving); + if (ret == RefcountWithFlags::incResult::incOk) { + ++handleCount_.tlStats(); + } + return ret; } template @@ -982,11 +987,16 @@ CacheAllocator::acquire(Item* it) { SCOPE_FAIL { stats_.numRefcountOverflow.inc(); }; - if (LIKELY(incRef(*it))) { + auto failIfMoving = getNumTiers() > 1; + auto incRes = incRef(*it, failIfMoving); + if (LIKELY(incRes == RefcountWithFlags::incResult::incOk)) { return WriteHandle{it, *this}; - } else { + } else if (incRes == RefcountWithFlags::incResult::incFailedEviction){ // item is being evicted return WriteHandle{}; + } else { + // item is being moved - wait for completion + return handleWithWaitContextForMovingItem(*it); } } @@ -1029,6 +1039,25 @@ bool CacheAllocator::replaceInMMContainer(Item& oldItem, } } +template +bool CacheAllocator::replaceInMMContainer(Item* oldItem, + Item& newItem) { + return replaceInMMContainer(*oldItem, newItem); +} + +template +bool CacheAllocator::replaceInMMContainer(EvictionIterator& oldItemIt, + Item& newItem) { + auto& oldContainer = getMMContainer(*oldItemIt); + auto& newContainer = getMMContainer(newItem); + + // This function is used for eviction across tiers + XDCHECK(&oldContainer != &newContainer); + oldContainer.remove(oldItemIt); + + return newContainer.add(newItem); +} + template bool CacheAllocator::replaceChainedItemInMMContainer( Item& oldItem, Item& newItem) { @@ -1174,6 +1203,143 @@ CacheAllocator::insertOrReplace(const WriteHandle& handle) { return replaced; } +/* Next two methods are used to asynchronously move Item between memory tiers. + * + * The thread, which moves Item, allocates new Item in the tier we are moving to + * and calls moveRegularItemWithSync() method. This method does the following: + * 1. Update the access container with the new item from the tier we are + * moving to. This Item has moving flag set. + * 2. Copy data from the old Item to the new one. + * + * Concurrent threads which are getting handle to the same key: + * 1. When a handle is created it checks if the moving flag is set + * 2. If so, Handle implementation creates waitContext and adds it to the + * MoveCtx by calling handleWithWaitContextForMovingItem() method. + * 3. Wait until the moving thread will complete its job. + */ +template +typename CacheAllocator::WriteHandle +CacheAllocator::handleWithWaitContextForMovingItem(Item& item) { + auto shard = getShardForKey(item.getKey()); + auto& movesMap = getMoveMapForShard(shard); + { + auto lock = getMoveLockForShard(shard); + + WriteHandle hdl{*this}; + auto waitContext = hdl.getItemWaitContext(); + + auto ret = movesMap.try_emplace(item.getKey(), std::make_unique()); + ret.first->second->addWaiter(std::move(waitContext)); + + return hdl; + } +} + +template +size_t CacheAllocator::wakeUpWaitersLocked(folly::StringPiece key, + WriteHandle&& handle) { + std::unique_ptr ctx; + auto shard = getShardForKey(key); + auto& movesMap = getMoveMapForShard(shard); + { + auto lock = getMoveLockForShard(shard); + movesMap.eraseInto(key, [&](auto &&key, auto &&value) { + ctx = std::move(value); + }); + } + + if (ctx) { + ctx->setItemHandle(std::move(handle)); + return ctx->numWaiters(); + } + + return 0; +} + +template +void CacheAllocator::moveRegularItemWithSync( + Item& oldItem, WriteHandle& newItemHdl) { + XDCHECK(oldItem.isMoving()); + XDCHECK(!oldItem.isExpired()); + // TODO: should we introduce new latency tracker. E.g. evictRegularLatency_ + // ??? util::LatencyTracker tracker{stats_.evictRegularLatency_}; + + XDCHECK_EQ(newItemHdl->getSize(), oldItem.getSize()); + + // take care of the flags before we expose the item to be accessed. this + // will ensure that when another thread removes the item from RAM, we issue + // a delete accordingly. See D7859775 for an example + if (oldItem.isNvmClean()) { + newItemHdl->markNvmClean(); + } + + // mark new item as moving to block readers until the data is copied + // (moveCb is called). Mark item in MMContainer temporarily (TODO: should + // we remove markMoving requirement for the item to be linked?) + newItemHdl->markInMMContainer(); + auto marked = newItemHdl->markMoving(false /* there is already a handle */); + newItemHdl->unmarkInMMContainer(); + XDCHECK(marked); + + auto predicate = [&](const Item& item){ + // we rely on moving flag being set (it should block all readers) + XDCHECK(item.getRefCount() == 0); + return true; + }; + + auto replaced = accessContainer_->replaceIf(oldItem, *newItemHdl, + predicate); + + if (config_.moveCb) { + // Execute the move callback. We cannot make any guarantees about the + // consistency of the old item beyond this point, because the callback can + // do more than a simple memcpy() e.g. update external references. If there + // are any remaining handles to the old item, it is the caller's + // responsibility to invalidate them. The move can only fail after this + // statement if the old item has been removed or replaced, in which case it + // should be fine for it to be left in an inconsistent state. + config_.moveCb(oldItem, *newItemHdl, nullptr); + } else { + std::memcpy(newItemHdl->getMemory(), oldItem.getMemory(), + oldItem.getSize()); + } + + // Adding the item to mmContainer has to succeed since no one can remove the item + auto& newContainer = getMMContainer(*newItemHdl); + auto mmContainerAdded = newContainer.add(*newItemHdl); + XDCHECK(mmContainerAdded); + + // no one can add or remove chained items at this point + if (oldItem.hasChainedItem()) { + // safe to acquire handle for a moving Item + auto incRes = incRef(oldItem, false); + XDCHECK(incRes == RefcountWithFlags::incResult::incOk); + auto oldHandle = WriteHandle{&oldItem,*this}; + XDCHECK_EQ(1u, oldHandle->getRefCount()) << oldHandle->toString(); + XDCHECK(!newItemHdl->hasChainedItem()) << newItemHdl->toString(); + try { + auto l = chainedItemLocks_.lockExclusive(oldItem.getKey()); + transferChainLocked(oldHandle, newItemHdl); + } catch (const std::exception& e) { + // this should never happen because we drained all the handles. + XLOGF(DFATAL, "{}", e.what()); + throw; + } + + XDCHECK(!oldItem.hasChainedItem()); + XDCHECK(newItemHdl->hasChainedItem()); + } + newItemHdl.unmarkNascent(); + auto ref = newItemHdl->unmarkMoving(); + //remove because there is a chance the new item was not + //added to the access container + if (UNLIKELY(ref == 0)) { + const auto res = + releaseBackToAllocator(*newItemHdl, RemoveContext::kNormal, false); + XDCHECK(res == ReleaseRes::kReleased); + } +} + template bool CacheAllocator::moveRegularItem(Item& oldItem, WriteHandle& newItemHdl) { @@ -1205,10 +1371,10 @@ bool CacheAllocator::moveRegularItem(Item& oldItem, config_.moveCb(oldItem, *newItemHdl, nullptr); // Inside the access container's lock, this checks if the old item is - // accessible and its refcount is zero. If the item is not accessible, + // accessible and its refcount is one. If the item is not accessible, // there is no point to replace it since it had already been removed // or in the process of being removed. If the item is in cache but the - // refcount is non-zero, it means user could be attempting to remove + // refcount is non-one, it means user could be attempting to remove // this item through an API such as remove(itemHandle). In this case, // it is unsafe to replace the old item with a new one, so we should // also abort. @@ -1234,13 +1400,12 @@ bool CacheAllocator::moveRegularItem(Item& oldItem, // no one can add or remove chained items at this point if (oldItem.hasChainedItem()) { - // safe to acquire handle for a moving Item - auto oldHandle = acquire(&oldItem); - XDCHECK_EQ(1u, oldHandle->getRefCount()) << oldHandle->toString(); + auto oldItemHdl = acquire(&oldItem); + XDCHECK_EQ(1u, oldItemHdl->getRefCount()) << oldItemHdl->toString(); XDCHECK(!newItemHdl->hasChainedItem()) << newItemHdl->toString(); try { auto l = chainedItemLocks_.lockExclusive(oldItem.getKey()); - transferChainLocked(oldHandle, newItemHdl); + transferChainLocked(oldItemHdl, newItemHdl); } catch (const std::exception& e) { // this should never happen because we drained all the handles. XLOGF(DFATAL, "{}", e.what()); @@ -1347,9 +1512,10 @@ CacheAllocator::getNextCandidate(TierId tid, Item* toRecycle = nullptr; Item* candidate = nullptr; auto& mmContainer = getMMContainer(tid, pid, cid); + bool lastTier = tid+1 >= getNumTiers(); mmContainer.withEvictionIterator([this, pid, cid, &candidate, &toRecycle, - &searchTries, &mmContainer, + &searchTries, &mmContainer, &lastTier, &token](auto&& itr) { if (!itr) { ++searchTries; @@ -1369,16 +1535,21 @@ CacheAllocator::getNextCandidate(TierId tid, ? &toRecycle_->asChainedItem().getParentItem(compressor_) : toRecycle_; - auto putToken = createPutToken(*candidate_); + typename NvmCacheT::PutToken putToken; + if (lastTier) { + // if it's last tier, the item will be evicted + // need to create put token before marking it exclusive + putToken = createPutToken(*candidate_); + } - if (shouldWriteToNvmCache(*candidate_) && !putToken.isValid()) { + if (lastTier && shouldWriteToNvmCache(*candidate_) && !putToken.isValid()) { stats_.evictFailConcurrentFill.inc(); ++itr; continue; } - auto markedForEviction = candidate_->markForEviction(); - if (!markedForEviction) { + auto marked = lastTier ? candidate_->markForEviction() : candidate_->markMoving(true); + if (!marked) { if (candidate_->hasChainedItem()) { stats_.evictFailParentAC.inc(); } else { @@ -1388,8 +1559,10 @@ CacheAllocator::getNextCandidate(TierId tid, continue; } + XDCHECK(candidate_->isMoving() || candidate_->isMarkedForEviction()); // markForEviction to make sure no other thead is evicting the item - // nor holding a handle to that item + // nor holding a handle to that item if this is last tier + // since we won't be moving the item to the next tier toRecycle = toRecycle_; candidate = candidate_; token = std::move(putToken); @@ -1412,13 +1585,44 @@ CacheAllocator::getNextCandidate(TierId tid, XDCHECK(toRecycle); XDCHECK(candidate); - XDCHECK(candidate->isMarkedForEviction()); - - unlinkItemForEviction(*candidate); + XDCHECK(candidate->isMoving() || candidate->isMarkedForEviction()); + + auto evictedToNext = lastTier ? nullptr + : tryEvictToNextMemoryTier(*candidate); + if (!evictedToNext) { + if (!token.isValid()) { + token = createPutToken(*candidate); + } + // tryEvictToNextMemoryTier should only fail if allocation of the new item fails + // in that case, it should be still possible to mark item as exclusive. + // + // in case that we are on the last tier, we whould have already marked + // as exclusive since we will not be moving the item to the next tier + // but rather just evicting all together, no need to + // markExclusiveWhenMoving + auto ret = lastTier ? true : candidate->markForEvictionWhenMoving(); + XDCHECK(ret); + + unlinkItemForEviction(*candidate); + // wake up any readers that wait for the move to complete + // it's safe to do now, as we have the item marked exclusive and + // no other reader can be added to the waiters list + wakeUpWaiters(*candidate, {}); + + if (token.isValid() && shouldWriteToNvmCacheExclusive(*candidate)) { + nvmCache_->put(*candidate, std::move(token)); + } + } else { + XDCHECK(!evictedToNext->isMarkedForEviction() && !evictedToNext->isMoving()); + XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); + XDCHECK(!candidate->isAccessible()); + XDCHECK(candidate->getKey() == evictedToNext->getKey()); - if (token.isValid() && shouldWriteToNvmCacheExclusive(*candidate)) { - nvmCache_->put(*candidate, std::move(token)); + wakeUpWaiters(*candidate, std::move(evictedToNext)); } + + XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); + return {candidate, toRecycle}; } @@ -1511,6 +1715,49 @@ bool CacheAllocator::shouldWriteToNvmCacheExclusive( return true; } +template +typename CacheAllocator::WriteHandle +CacheAllocator::tryEvictToNextMemoryTier( + TierId tid, PoolId pid, Item& item) { + XDCHECK(item.isMoving()); + XDCHECK(item.getRefCount() == 0); + if(item.hasChainedItem()) return WriteHandle{}; // TODO: We do not support ChainedItem yet + if(item.isExpired()) { + accessContainer_->remove(item); + item.unmarkMoving(); + return acquire(&item); + } + + TierId nextTier = tid; // TODO - calculate this based on some admission policy + while (++nextTier < getNumTiers()) { // try to evict down to the next memory tiers + // allocateInternal might trigger another eviction + auto newItemHdl = allocateInternalTier(nextTier, pid, + item.getKey(), + item.getSize(), + item.getCreationTime(), + item.getExpiryTime()); + + if (newItemHdl) { + XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); + moveRegularItemWithSync(item, newItemHdl); + item.unmarkMoving(); + return newItemHdl; + } else { + return WriteHandle{}; + } + } + + return {}; +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::tryEvictToNextMemoryTier(Item& item) { + auto tid = getTierId(item); + auto pid = allocator_[tid]->getAllocInfo(item.getMemory()).poolId; + return tryEvictToNextMemoryTier(tid, pid, item); +} + template typename CacheAllocator::RemoveRes CacheAllocator::remove(typename Item::Key key) { @@ -2041,8 +2288,7 @@ std::vector CacheAllocator::dumpEvictionIterator( std::vector content; - size_t i = 0; - while (i < numItems && tid >= 0) { + while (tid >= 0) { auto& mm = *mmContainers_[tid][pid][cid]; mm.withEvictionIterator([&content, numItems](auto&& itr) { while (itr && content.size() < numItems) { @@ -2052,7 +2298,6 @@ std::vector CacheAllocator::dumpEvictionIterator( }); --tid; } - return content; } @@ -2636,6 +2881,14 @@ void CacheAllocator::throttleWith(util::Throttler& t, } } +template +typename RefcountWithFlags::Value CacheAllocator::unmarkMovingAndWakeUpWaiters(Item &item, WriteHandle handle) +{ + auto ret = item.unmarkMoving(); + wakeUpWaiters(item, std::move(handle)); + return ret; +} + template bool CacheAllocator::moveForSlabRelease( const SlabReleaseContext& ctx, Item& oldItem, util::Throttler& throttler) { @@ -2654,7 +2907,8 @@ bool CacheAllocator::moveForSlabRelease( // Nothing to move and the key is likely also bogus for chained items. if (oldItem.isOnlyMoving()) { - oldItem.unmarkMoving(); + auto ret = unmarkMovingAndWakeUpWaiters(oldItem, {}); + XDCHECK(ret == 0); const auto res = releaseBackToAllocator(oldItem, RemoveContext::kNormal, false); XDCHECK(res == ReleaseRes::kReleased); @@ -2702,8 +2956,9 @@ bool CacheAllocator::moveForSlabRelease( }); } auto tid = getTierId(oldItem); - auto ref = oldItem.unmarkMoving(); - XDCHECK_EQ(ref, 0); + auto ref = unmarkMovingAndWakeUpWaiters(oldItem, std::move(newItemHdl)); + XDCHECK(ref == 0); + const auto allocInfo = allocator_[tid]->getAllocInfo(oldItem.getMemory()); allocator_[tid]->free(&oldItem); @@ -2753,12 +3008,12 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { // Set up the destination for the move. Since oldChainedItem would // be marked as moving, it won't be picked for eviction. auto newItemHdl = - allocateChainedItemInternal(parentHandle, oldChainedItem.getSize()); + allocateChainedItemInternal(parentHandle, oldItem.getSize()); if (!newItemHdl) { return {}; } - XDCHECK_EQ(newItemHdl->getSize(), oldChainedItem.getSize()); + XDCHECK_EQ(newItemHdl->getSize(), oldItem.getSize()); auto parentPtr = parentHandle.getInternal(); XDCHECK_EQ(reinterpret_cast(parentPtr), reinterpret_cast( @@ -2821,21 +3076,44 @@ bool CacheAllocator::tryMovingForSlabRelease( } } - return oldItem.isChainedItem() - ? moveChainedItem(oldItem.asChainedItem(), newItemHdl) - : moveRegularItem(oldItem, newItemHdl); + // TODO: we can unify move*Item and move*ItemWithSync by always + // using the moving bit to block readers. + if (getNumTiers() == 1) { + return oldItem.isChainedItem() + ? moveChainedItem(oldItem.asChainedItem(), newItemHdl) + : moveRegularItem(oldItem, newItemHdl); + } else { + if (oldItem.isChainedItem() || oldItem.hasChainedItem()) { + // TODO: add support for chained items + return false; + } else { + moveRegularItemWithSync(oldItem, newItemHdl); + removeFromMMContainer(oldItem); + return true; + } + } +} + +template +void CacheAllocator::wakeUpWaiters(Item& item, WriteHandle handle) +{ + // readers do not block on 'moving' items in case there is only one tier + if (getNumTiers() > 1) { + wakeUpWaitersLocked(item.getKey(), std::move(handle)); + } } template void CacheAllocator::evictForSlabRelease( const SlabReleaseContext& ctx, Item& item, util::Throttler& throttler) { auto startTime = util::getCurrentTimeSec(); + while (true) { XDCHECK(item.isMoving()); stats_.numEvictionAttempts.inc(); if (shutDownInProgress_) { - item.unmarkMoving(); + auto ref = unmarkMovingAndWakeUpWaiters(item, {}); allocator_[getTierId(item)]->abortSlabRelease(ctx); throw exception::SlabReleaseAborted( folly::sformat("Slab Release aborted while trying to evict" @@ -2855,12 +3133,12 @@ void CacheAllocator::evictForSlabRelease( .toString()) : ""); }); - // if the item is already in a state where only the exclusive bit is set, // nothing needs to be done. We simply need to call unmarkMoving and free // the item. if (item.isOnlyMoving()) { - item.unmarkMoving(); + auto ref = unmarkMovingAndWakeUpWaiters(item, {}); + XDCHECK(ref == 0); const auto res = releaseBackToAllocator(item, RemoveContext::kNormal, false); XDCHECK(ReleaseRes::kReleased == res); @@ -2917,6 +3195,10 @@ void CacheAllocator::evictForSlabRelease( // unmark the child so it will be freed item.unmarkMoving(); unlinkItemForEviction(*evicted); + // wake up any readers that wait for the move to complete + // it's safe to do now, as we have the item marked exclusive and + // no other reader can be added to the waiters list + wakeUpWaiters(*evicted, {}); } else { continue; } @@ -2926,6 +3208,7 @@ void CacheAllocator::evictForSlabRelease( token = createPutToken(*evicted); if (evicted->markForEvictionWhenMoving()) { unlinkItemForEviction(*evicted); + wakeUpWaiters(*evicted, {}); } else { continue; } @@ -2936,7 +3219,7 @@ void CacheAllocator::evictForSlabRelease( } const auto allocInfo = - allocator_[getTierId(item)]->getAllocInfo(static_cast(evicted)); + allocator_[getTierId(*evicted)]->getAllocInfo(static_cast(evicted)); if (evicted->hasChainedItem()) { (*stats_.chainedItemEvictions)[allocInfo.poolId][allocInfo.classId].inc(); } else { @@ -2953,6 +3236,21 @@ void CacheAllocator::evictForSlabRelease( } } +template +template +typename CacheAllocator::WriteHandle +CacheAllocator::removeIf(Item& item, Fn&& predicate) { + auto handle = accessContainer_->removeIf(item, std::forward(predicate)); + + if (handle) { + XDCHECK_EQ(reinterpret_cast(handle.get()), + reinterpret_cast(&item)); + removeFromMMContainer(item); + } + + return handle; +} + template bool CacheAllocator::removeIfExpired(const ReadHandle& handle) { if (!handle) { @@ -2961,14 +3259,7 @@ bool CacheAllocator::removeIfExpired(const ReadHandle& handle) { // We remove the item from both access and mm containers. // We want to make sure the caller is the only one holding the handle. - auto removedHandle = - accessContainer_->removeIf(*(handle.getInternal()), itemExpiryPredicate); - if (removedHandle) { - removeFromMMContainer(*(handle.getInternal())); - return true; - } - - return false; + return (bool)removeIf(*(handle.getInternal()), itemExpiryPredicate); } template @@ -2987,15 +3278,14 @@ bool CacheAllocator::markMovingForSlabRelease( // At first, we assume this item was already freed bool itemFreed = true; bool markedMoving = false; - TierId tid = 0; - const auto fn = [&markedMoving, &itemFreed, &tid, this /* TODO - necessary for getTierId */](void* memory) { + TierId tid = getTierId(alloc); + const auto fn = [&markedMoving, &itemFreed](void* memory) { // Since this callback is executed, the item is not yet freed itemFreed = false; Item* item = static_cast(memory); - if (item->markMoving()) { + if (item->markMoving(false)) { markedMoving = true; } - tid = getTierId(*item); }; auto startTime = util::getCurrentTimeSec(); diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 145f1e3dca..bc228d1c5b 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -1326,7 +1328,7 @@ class CacheAllocator : public CacheBase { private: // wrapper around Item's refcount and active handle tracking - FOLLY_ALWAYS_INLINE bool incRef(Item& it); + FOLLY_ALWAYS_INLINE RefcountWithFlags::incResult incRef(Item& it, bool failIfMoving); FOLLY_ALWAYS_INLINE RefcountWithFlags::Value decRef(Item& it); // drops the refcount and if needed, frees the allocation back to the memory @@ -1556,7 +1558,7 @@ class CacheAllocator : public CacheBase { // // @return true If the move was completed, and the containers were updated // successfully. - bool moveRegularItemOnEviction(Item& oldItem, WriteHandle& newItemHdl); + void moveRegularItemWithSync(Item& oldItem, WriteHandle& newItemHdl); // Moves a regular item to a different slab. This should only be used during // slab release after the item's exclusive bit has been set. The user supplied @@ -1643,6 +1645,10 @@ class CacheAllocator : public CacheBase { // false if the item is not in MMContainer bool removeFromMMContainer(Item& item); + using EvictionIterator = typename MMContainer::LockedIterator; + + WriteHandle acquire(EvictionIterator& it) { return acquire(it.get()); } + // Replaces an item in the MMContainer with another item, at the same // position. // @@ -1653,6 +1659,8 @@ class CacheAllocator : public CacheBase { // destination item did not exist in the container, or if the // source item already existed. bool replaceInMMContainer(Item& oldItem, Item& newItem); + bool replaceInMMContainer(Item* oldItem, Item& newItem); + bool replaceInMMContainer(EvictionIterator& oldItemIt, Item& newItem); // Replaces an item in the MMContainer with another item, at the same // position. Or, if the two chained items belong to two different MM @@ -1735,7 +1743,35 @@ class CacheAllocator : public CacheBase { ClassId cid, unsigned int& searchTries); - using EvictionIterator = typename MMContainer::LockedIterator; + // Try to move the item down to the next memory tier + // + // @param tid current tier ID of the item + // @param pid the pool ID the item belong to. + // @param item the item to evict + // + // @return valid handle to the item. This will be the last + // handle to the item. On failure an empty handle. + WriteHandle tryEvictToNextMemoryTier(TierId tid, PoolId pid, Item& item); + + // Wakes up waiters if there are any + // + // @param item wakes waiters that are waiting on that item + // @param handle handle to pass to the waiters + void wakeUpWaiters(Item& item, WriteHandle handle); + + // Unmarks item as moving and wakes up any waiters waiting on that item + // + // @param item wakes waiters that are waiting on that item + // @param handle handle to pass to the waiters + typename RefcountWithFlags::Value unmarkMovingAndWakeUpWaiters(Item &item, WriteHandle handle); + + // Try to move the item down to the next memory tier + // + // @param item the item to evict + // + // @return valid handle to the item. This will be the last + // handle to the item. On failure an empty handle. + WriteHandle tryEvictToNextMemoryTier(Item& item); // Deserializer CacheAllocatorMetadata and verify the version // @@ -1853,6 +1889,12 @@ class CacheAllocator : public CacheBase { typename NvmCacheT::PutToken createPutToken(Item& item); + // Helper function to remove a item if predicates is true. + // + // @return last handle to the item on success. empty handle on failure. + template + WriteHandle removeIf(Item& item, Fn&& predicate); + // Helper function to remove a item if expired. // // @return true if it item expire and removed successfully. @@ -2038,6 +2080,87 @@ class CacheAllocator : public CacheBase { size_t memoryTierSize(TierId tid) const; + WriteHandle handleWithWaitContextForMovingItem(Item& item); + + size_t wakeUpWaitersLocked(folly::StringPiece key, WriteHandle&& handle); + + class MoveCtx { + public: + MoveCtx() {} + + ~MoveCtx() { + // prevent any further enqueue to waiters + // Note: we don't need to hold locks since no one can enqueue + // after this point. + wakeUpWaiters(); + } + + // record the item handle. Upon destruction we will wake up the waiters + // and pass a clone of the handle to the callBack. By default we pass + // a null handle + void setItemHandle(WriteHandle _it) { it = std::move(_it); } + + // enqueue a waiter into the waiter list + // @param waiter WaitContext + void addWaiter(std::shared_ptr> waiter) { + XDCHECK(waiter); + waiters.push_back(std::move(waiter)); + } + + size_t numWaiters() const { return waiters.size(); } + + private: + // notify all pending waiters that are waiting for the fetch. + void wakeUpWaiters() { + bool refcountOverflowed = false; + for (auto& w : waiters) { + // If refcount overflowed earlier, then we will return miss to + // all subsequent waitors. + if (refcountOverflowed) { + w->set(WriteHandle{}); + continue; + } + + try { + w->set(it.clone()); + } catch (const exception::RefcountOverflow&) { + // We'll return a miss to the user's pending read, + // so we should enqueue a delete via NvmCache. + // TODO: cache.remove(it); + refcountOverflowed = true; + } + } + } + + WriteHandle it; // will be set when Context is being filled + std::vector>> waiters; // list of + // waiters + }; + using MoveMap = + folly::F14ValueMap, + folly::HeterogeneousAccessHash>; + + static size_t getShardForKey(folly::StringPiece key) { + return folly::Hash()(key) % kShards; + } + + MoveMap& getMoveMapForShard(size_t shard) { + return movesMap_[shard].movesMap_; + } + + MoveMap& getMoveMap(folly::StringPiece key) { + return getMoveMapForShard(getShardForKey(key)); + } + + std::unique_lock getMoveLockForShard(size_t shard) { + return std::unique_lock(moveLock_[shard].moveLock_); + } + + std::unique_lock getMoveLock(folly::StringPiece key) { + return getMoveLockForShard(getShardForKey(key)); + } + // Whether the memory allocator for this cache allocator was created on shared // memory. The hash table, chained item hash table etc is also created on // shared memory except for temporary shared memory mode when they're created @@ -2128,6 +2251,22 @@ class CacheAllocator : public CacheBase { // poolResizer_, poolOptimizer_, memMonitor_, reaper_ mutable std::mutex workersMutex_; + static constexpr size_t kShards = 8192; // TODO: need to define right value + + struct MovesMapShard { + alignas(folly::hardware_destructive_interference_size) MoveMap movesMap_; + }; + + struct MoveLock { + alignas(folly::hardware_destructive_interference_size) std::mutex moveLock_; + }; + + // a map of all pending moves + std::vector movesMap_; + + // a map of move locks for each shard + std::vector moveLock_; + // time when the ram cache was first created const uint32_t cacheCreationTime_{0}; diff --git a/cachelib/allocator/CacheItem-inl.h b/cachelib/allocator/CacheItem-inl.h index bf77b43aa5..b3585f7a35 100644 --- a/cachelib/allocator/CacheItem-inl.h +++ b/cachelib/allocator/CacheItem-inl.h @@ -238,8 +238,8 @@ bool CacheItem::markForEvictionWhenMoving() { } template -bool CacheItem::markMoving() { - return ref_.markMoving(); +bool CacheItem::markMoving(bool failIfRefNotZero) { + return ref_.markMoving(failIfRefNotZero); } template diff --git a/cachelib/allocator/CacheItem.h b/cachelib/allocator/CacheItem.h index afee315cbb..b4fa339b57 100644 --- a/cachelib/allocator/CacheItem.h +++ b/cachelib/allocator/CacheItem.h @@ -309,9 +309,9 @@ class CACHELIB_PACKED_ATTR CacheItem { // // @return true on success, failure if item is marked as exclusive // @throw exception::RefcountOverflow on ref count overflow - FOLLY_ALWAYS_INLINE bool incRef() { + FOLLY_ALWAYS_INLINE RefcountWithFlags::incResult incRef(bool failIfMoving) { try { - return ref_.incRef(); + return ref_.incRef(failIfMoving); } catch (exception::RefcountOverflow& e) { throw exception::RefcountOverflow( folly::sformat("{} item: {}", e.what(), toString())); @@ -378,7 +378,7 @@ class CACHELIB_PACKED_ATTR CacheItem { * Unmarking moving will also return the refcount at the moment of * unmarking. */ - bool markMoving(); + bool markMoving(bool failIfRefNotZero); RefcountWithFlags::Value unmarkMoving() noexcept; bool isMoving() const noexcept; bool isOnlyMoving() const noexcept; diff --git a/cachelib/allocator/Handle.h b/cachelib/allocator/Handle.h index 11d2bed2be..06c21bffe4 100644 --- a/cachelib/allocator/Handle.h +++ b/cachelib/allocator/Handle.h @@ -400,6 +400,12 @@ struct ReadHandleImpl { } } + protected: + friend class ReadHandleImpl; + // Method used only by ReadHandleImpl ctor + void discard() { + it_.store(nullptr, std::memory_order_relaxed); + } private: // we are waiting on Item* to be set to a value. One of the valid values is // nullptr. So choose something that we dont expect to indicate a ptr @@ -479,7 +485,8 @@ struct ReadHandleImpl { // Handle which has the item already FOLLY_ALWAYS_INLINE ReadHandleImpl(Item* it, CacheT& alloc) noexcept - : alloc_(&alloc), it_(it) {} + : alloc_(&alloc), it_(it) { + } // handle that has a wait context allocated. Used for async handles // In this case, the it_ will be filled in asynchronously and mulitple diff --git a/cachelib/allocator/MMLru-inl.h b/cachelib/allocator/MMLru-inl.h index d35759f212..842d87ddb8 100644 --- a/cachelib/allocator/MMLru-inl.h +++ b/cachelib/allocator/MMLru-inl.h @@ -229,6 +229,18 @@ void MMLru::Container::withEvictionIterator(F&& fun) { } } +//template T::*HookPtr> +//template +//void +//MMLru::Container::withPromotionIterator(F&& fun) { +// if (config_.useCombinedLockForIterators) { +// lruMutex_->lock_combine([this, &fun]() { fun(Iterator{lru_.begin()}); }); +// } else { +// LockHolder lck{*lruMutex_}; +// fun(Iterator{lru_.begin()}); +// } +//} + template T::*HookPtr> void MMLru::Container::ensureNotInsertionPoint(T& node) noexcept { // If we are removing the insertion point node, grow tail before we remove diff --git a/cachelib/allocator/MMLru.h b/cachelib/allocator/MMLru.h index 29c6d02689..645b8f0e86 100644 --- a/cachelib/allocator/MMLru.h +++ b/cachelib/allocator/MMLru.h @@ -230,12 +230,13 @@ class MMLru { // lruInsertionPointSpec = 2, we insert at a point 1/4th from tail uint8_t lruInsertionPointSpec{0}; + // Whether to use combined locking for withEvictionIterator. + bool useCombinedLockForIterators{true}; + // Minimum interval between reconfigurations. If 0, reconfigure is never // called. std::chrono::seconds mmReconfigureIntervalSecs{}; - // Whether to use combined locking for withEvictionIterator. - bool useCombinedLockForIterators{false}; }; // The container object which can be used to keep track of objects of type diff --git a/cachelib/allocator/Refcount.h b/cachelib/allocator/Refcount.h index 107e10735e..902f217760 100644 --- a/cachelib/allocator/Refcount.h +++ b/cachelib/allocator/Refcount.h @@ -130,30 +130,41 @@ class FOLLY_PACK_ATTR RefcountWithFlags { RefcountWithFlags& operator=(const RefcountWithFlags&) = delete; RefcountWithFlags(RefcountWithFlags&&) = delete; RefcountWithFlags& operator=(RefcountWithFlags&&) = delete; - + enum incResult { + incOk, + incFailedMoving, + incFailedEviction + }; // Bumps up the reference count only if the new count will be strictly less // than or equal to the maxCount and the item is not exclusive // @return true if refcount is bumped. false otherwise (if item is exclusive) // @throw exception::RefcountOverflow if new count would be greater than // maxCount - FOLLY_ALWAYS_INLINE bool incRef() { - auto predicate = [](const Value curValue) { - Value bitMask = getAdminRef(); - - const bool exlusiveBitIsSet = curValue & bitMask; - if (UNLIKELY((curValue & kAccessRefMask) == (kAccessRefMask))) { - throw exception::RefcountOverflow("Refcount maxed out."); - } - - // Check if the item is not marked for eviction - return !exlusiveBitIsSet || ((curValue & kAccessRefMask) != 0); - }; - - auto newValue = [](const Value curValue) { - return (curValue + static_cast(1)); - }; - - return atomicUpdateValue(predicate, newValue); + FOLLY_ALWAYS_INLINE incResult incRef(bool failIfMoving) { + incResult res = incOk; + auto predicate = [failIfMoving, &res](const Value curValue) { + Value bitMask = getAdminRef(); + + const bool exlusiveBitIsSet = curValue & bitMask; + if (UNLIKELY((curValue & kAccessRefMask) == (kAccessRefMask))) { + throw exception::RefcountOverflow("Refcount maxed out."); + } else if (exlusiveBitIsSet && (curValue & kAccessRefMask) == 0) { + res = incFailedEviction; + return false; + } else if (exlusiveBitIsSet && failIfMoving) { + res = incFailedMoving; + return false; + } + res = incOk; + return true; + }; + + auto newValue = [](const Value curValue) { + return (curValue + static_cast(1)); + }; + + atomicUpdateValue(predicate, newValue); + return res; } // Bumps down the reference count @@ -319,14 +330,16 @@ class FOLLY_PACK_ATTR RefcountWithFlags { * Unmarking clears `kExclusive` bit and decreses the interanl refCount by 1. * `unmarkMoving` does does not depend on `isInMMContainer` */ - bool markMoving() { + bool markMoving(bool failIfRefNotZero) { Value linkedBitMask = getAdminRef(); Value exclusiveBitMask = getAdminRef(); - auto predicate = [linkedBitMask, exclusiveBitMask](const Value curValue) { + auto predicate = [failIfRefNotZero, linkedBitMask, exclusiveBitMask](const Value curValue) { const bool unlinked = !(curValue & linkedBitMask); const bool alreadyExclusive = curValue & exclusiveBitMask; - + if (failIfRefNotZero && (curValue & kAccessRefMask) != 0) { + return false; + } if (unlinked || alreadyExclusive) { return false; } diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp index c56f640847..3b7d9bdf37 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp @@ -26,6 +26,8 @@ using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest; TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersRemoveDuringEviction) { this->testMultiTiersRemoveDuringEviction(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersReplaceDuringEviction) { this->testMultiTiersReplaceDuringEviction(); } } // end of namespace tests } // end of namespace cachelib diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h index 2ecb2c14ca..c336b59a4b 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h @@ -20,12 +20,42 @@ #include "cachelib/allocator/MemoryTierCacheConfig.h" #include "cachelib/allocator/tests/TestBase.h" +#include + namespace facebook { namespace cachelib { namespace tests { template class AllocatorMemoryTiersTest : public AllocatorTest { + private: + template + void testMultiTiersAsyncOpDuringMove(std::unique_ptr& alloc, + PoolId& pool, bool& quit, MvCallback&& moveCb) { + typename AllocatorT::Config config; + config.setCacheSize(4 * Slab::kSize); + config.enableCachePersistence("/tmp"); + config.configureMemoryTiers({ + MemoryTierCacheConfig::fromShm() + .setRatio(1).setMemBind(std::string("0")), + MemoryTierCacheConfig::fromShm() + .setRatio(1).setMemBind(std::string("0")) + }); + + config.enableMovingOnSlabRelease(moveCb, {} /* ChainedItemsMoveSync */, + -1 /* movingAttemptsLimit */); + + alloc = std::make_unique(AllocatorT::SharedMemNew, config); + ASSERT(alloc != nullptr); + pool = alloc->addPool("default", alloc->getCacheMemoryStats().ramCacheSize); + + int i = 0; + while(!quit) { + auto handle = alloc->allocate(pool, std::to_string(++i), std::string("value").size()); + ASSERT(handle != nullptr); + ASSERT_NO_THROW(alloc->insertOrReplace(handle)); + } + } public: void testMultiTiersInvalid() { typename AllocatorT::Config config; @@ -74,6 +104,70 @@ class AllocatorMemoryTiersTest : public AllocatorTest { ASSERT(handle != nullptr); ASSERT_NO_THROW(alloc->insertOrReplace(handle)); } + + void testMultiTiersRemoveDuringEviction() { + std::unique_ptr alloc; + PoolId pool; + std::unique_ptr t; + folly::Latch latch(1); + bool quit = false; + + auto moveCb = [&] (typename AllocatorT::Item& oldItem, + typename AllocatorT::Item& newItem, + typename AllocatorT::Item* /* parentPtr */) { + + auto key = oldItem.getKey(); + t = std::make_unique([&](){ + // remove() function is blocked by wait context + // till item is moved to next tier. So that, we should + // notify latch before calling remove() + latch.count_down(); + alloc->remove(key); + }); + // wait till async thread is running + latch.wait(); + memcpy(newItem.getMemory(), oldItem.getMemory(), oldItem.getSize()); + quit = true; + }; + + testMultiTiersAsyncOpDuringMove(alloc, pool, quit, moveCb); + + t->join(); + } + + void testMultiTiersReplaceDuringEviction() { + std::unique_ptr alloc; + PoolId pool; + std::unique_ptr t; + folly::Latch latch(1); + bool quit = false; + + auto moveCb = [&] (typename AllocatorT::Item& oldItem, + typename AllocatorT::Item& newItem, + typename AllocatorT::Item* /* parentPtr */) { + auto key = oldItem.getKey(); + if(!quit) { + // we need to replace only once because subsequent allocate calls + // will cause evictions recursevly + quit = true; + t = std::make_unique([&](){ + auto handle = alloc->allocate(pool, key, std::string("new value").size()); + // insertOrReplace() function is blocked by wait context + // till item is moved to next tier. So that, we should + // notify latch before calling insertOrReplace() + latch.count_down(); + ASSERT_NO_THROW(alloc->insertOrReplace(handle)); + }); + // wait till async thread is running + latch.wait(); + } + memcpy(newItem.getMemory(), oldItem.getMemory(), oldItem.getSize()); + }; + + testMultiTiersAsyncOpDuringMove(alloc, pool, quit, moveCb); + + t->join(); + } }; } // namespace tests } // namespace cachelib diff --git a/cachelib/allocator/tests/ItemHandleTest.cpp b/cachelib/allocator/tests/ItemHandleTest.cpp index d992a84011..5213166816 100644 --- a/cachelib/allocator/tests/ItemHandleTest.cpp +++ b/cachelib/allocator/tests/ItemHandleTest.cpp @@ -39,6 +39,8 @@ struct TestItem { using ChainedItem = int; void reset() {} + + folly::StringPiece getKey() const { return folly::StringPiece(); } }; struct TestNvmCache; @@ -80,6 +82,12 @@ struct TestAllocator { void adjustHandleCountForThread_private(int i) { tlRef_.tlStats() += i; } + bool addWaitContextForMovingItem( + folly::StringPiece key, + std::shared_ptr> waiter) { + return false; + } + util::FastStats tlRef_; }; } // namespace diff --git a/cachelib/allocator/tests/ItemTest.cpp b/cachelib/allocator/tests/ItemTest.cpp index 70dd1277fe..2f25cc07f0 100644 --- a/cachelib/allocator/tests/ItemTest.cpp +++ b/cachelib/allocator/tests/ItemTest.cpp @@ -82,8 +82,10 @@ TEST(ItemTest, ExpiryTime) { EXPECT_TRUE(result); EXPECT_EQ(tenMins, item->getConfiguredTTL()); + // So that exclusive bit will be set + item->markAccessible(); // Test that writes fail while the item is moving - result = item->markMoving(); + result = item->markMoving(true); EXPECT_TRUE(result); result = item->updateExpiryTime(0); EXPECT_FALSE(result); diff --git a/cachelib/allocator/tests/RefCountTest.cpp b/cachelib/allocator/tests/RefCountTest.cpp index 1f31894ddc..8f8ae9609a 100644 --- a/cachelib/allocator/tests/RefCountTest.cpp +++ b/cachelib/allocator/tests/RefCountTest.cpp @@ -52,7 +52,7 @@ void RefCountTest::testMultiThreaded() { nLocalRef--; ref.markAccessible(); } else { - ref.incRef(); + ref.incRef(true); nLocalRef++; ref.unmarkAccessible(); } @@ -101,12 +101,12 @@ void RefCountTest::testBasic() { ASSERT_FALSE(ref.template isFlagSet()); for (uint32_t i = 0; i < RefcountWithFlags::kAccessRefMask; i++) { - ASSERT_TRUE(ref.incRef()); + ASSERT_EQ(ref.incRef(true),RefcountWithFlags::incOk); } // Incrementing past the max will fail auto rawRef = ref.getRaw(); - ASSERT_THROW(ref.incRef(), std::overflow_error); + ASSERT_THROW(ref.incRef(true), std::overflow_error); ASSERT_EQ(rawRef, ref.getRaw()); // Bumping up access ref shouldn't affect admin ref and flags @@ -152,11 +152,11 @@ void RefCountTest::testBasic() { ASSERT_FALSE(ref.template isFlagSet()); // conditionally set flags - ASSERT_FALSE((ref.markMoving())); + ASSERT_FALSE(ref.markMoving(true)); ref.markInMMContainer(); // only first one succeeds - ASSERT_TRUE((ref.markMoving())); - ASSERT_FALSE((ref.markMoving())); + ASSERT_TRUE(ref.markMoving(true)); + ASSERT_FALSE(ref.markMoving(true)); ref.unmarkInMMContainer(); ref.template setFlag(); @@ -193,7 +193,7 @@ void RefCountTest::testMarkForEvictionAndMoving() { RefcountWithFlags ref; ref.markInMMContainer(); - ASSERT_TRUE(ref.markMoving()); + ASSERT_TRUE(ref.markMoving(true)); ASSERT_FALSE(ref.markForEviction()); ref.unmarkInMMContainer(); @@ -207,7 +207,7 @@ void RefCountTest::testMarkForEvictionAndMoving() { ref.markInMMContainer(); ASSERT_TRUE(ref.markForEviction()); - ASSERT_FALSE(ref.markMoving()); + ASSERT_FALSE(ref.markMoving(true)); ref.unmarkInMMContainer(); auto ret = ref.unmarkForEviction(); @@ -219,9 +219,9 @@ void RefCountTest::testMarkForEvictionAndMoving() { RefcountWithFlags ref; ref.markInMMContainer(); - ref.incRef(); + ref.incRef(true); - ASSERT_TRUE(ref.markMoving()); + ASSERT_TRUE(ref.markMoving(false)); ref.unmarkInMMContainer(); auto ret = ref.unmarkMoving(); @@ -233,7 +233,7 @@ void RefCountTest::testMarkForEvictionAndMoving() { RefcountWithFlags ref; ref.markInMMContainer(); - ref.incRef(); + ref.incRef(true); ASSERT_FALSE(ref.markForEviction()); } } diff --git a/cachelib/cachebench/util/CacheConfig.h b/cachelib/cachebench/util/CacheConfig.h index 13c0e1e7dc..0240fc4160 100644 --- a/cachelib/cachebench/util/CacheConfig.h +++ b/cachelib/cachebench/util/CacheConfig.h @@ -92,7 +92,7 @@ struct CacheConfig : public JSONConfig { bool lruUpdateOnWrite{false}; bool lruUpdateOnRead{true}; bool tryLockUpdate{false}; - bool useCombinedLockForIterators{false}; + bool useCombinedLockForIterators{true}; // LRU param uint64_t lruIpSpec{0}; From ed7b70f1ccb3f82941925a2027ebf3224905e703 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Thu, 30 Dec 2021 17:18:29 -0500 Subject: [PATCH 12/31] basic multi-tier test based on numa bindings --- .../allocator/tests/AllocatorTypeTest.cpp | 1 + cachelib/allocator/tests/BaseAllocatorTest.h | 80 +++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/cachelib/allocator/tests/AllocatorTypeTest.cpp b/cachelib/allocator/tests/AllocatorTypeTest.cpp index 1e98af29f2..a572485e8d 100644 --- a/cachelib/allocator/tests/AllocatorTypeTest.cpp +++ b/cachelib/allocator/tests/AllocatorTypeTest.cpp @@ -409,6 +409,7 @@ TYPED_TEST(BaseAllocatorTest, RateMap) { this->testRateMap(); } TYPED_TEST(BaseAllocatorTest, StatSnapshotTest) { this->testStatSnapshotTest(); } +TYPED_TEST(BaseAllocatorTest, BasicMultiTier) {this->testBasicMultiTier(); } namespace { // the tests that cannot be done by TYPED_TEST. diff --git a/cachelib/allocator/tests/BaseAllocatorTest.h b/cachelib/allocator/tests/BaseAllocatorTest.h index 820edb8478..e059972eb4 100644 --- a/cachelib/allocator/tests/BaseAllocatorTest.h +++ b/cachelib/allocator/tests/BaseAllocatorTest.h @@ -6316,6 +6316,86 @@ class BaseAllocatorTest : public AllocatorTest { }); EXPECT_EQ(intervalNameExists, 4); } + + void testSingleTierMemoryAllocatorSize() { + typename AllocatorT::Config config; + static constexpr size_t cacheSize = 100 * 1024 * 1024; /* 100 MB */ + config.setCacheSize(cacheSize); + config.enableCachePersistence(folly::sformat("/tmp/single-tier-test/{}", ::getpid())); + + AllocatorT alloc(AllocatorT::SharedMemNew, config); + + EXPECT_LE(alloc.allocator_[0]->getMemorySize(), cacheSize); + } + + void testSingleTierMemoryAllocatorSizeAnonymous() { + typename AllocatorT::Config config; + static constexpr size_t cacheSize = 100 * 1024 * 1024; /* 100 MB */ + config.setCacheSize(cacheSize); + + AllocatorT alloc(config); + + EXPECT_LE(alloc.allocator_[0]->getMemorySize(), cacheSize); + } + + void testBasicMultiTier() { + using Item = typename AllocatorT::Item; + const static std::string data = "data"; + + std::set movedKeys; + auto moveCb = [&](const Item& oldItem, Item& newItem, Item* /* parentPtr */) { + std::memcpy(newItem.getMemory(), oldItem.getMemory(), oldItem.getSize()); + movedKeys.insert(oldItem.getKey().str()); + }; + + typename AllocatorT::Config config; + static constexpr size_t cacheSize = 100 * 1024 * 1024; /* 100 MB */ + config.setCacheSize(100 * 1024 * 1024); /* 100 MB */ + config.enableCachePersistence(folly::sformat("/tmp/multi-tier-test/{}", ::getpid())); + config.configureMemoryTiers({ + MemoryTierCacheConfig::fromShm().setRatio(1) + .setMemBind(std::string("0")), + MemoryTierCacheConfig::fromShm().setRatio(1) + .setMemBind(std::string("0")), + }); + config.enableMovingOnSlabRelease(moveCb); + + AllocatorT alloc(AllocatorT::SharedMemNew, config); + + EXPECT_EQ(alloc.allocator_.size(), 2); + EXPECT_LE(alloc.allocator_[0]->getMemorySize(), cacheSize / 2); + EXPECT_LE(alloc.allocator_[1]->getMemorySize(), cacheSize / 2); + + const size_t numBytes = alloc.getCacheMemoryStats().ramCacheSize; + auto pid = alloc.addPool("default", numBytes); + + static constexpr size_t numOps = cacheSize / 1024; + for (int i = 0; i < numOps; i++) { + std::string key = std::to_string(i); + auto h = alloc.allocate(pid, key, 1024); + EXPECT_TRUE(h); + + std::memcpy(h->getMemory(), data.data(), data.size()); + + alloc.insertOrReplace(h); + } + + EXPECT_TRUE(movedKeys.size() > 0); + + size_t movedButStillInMemory = 0; + for (const auto &k : movedKeys) { + auto h = alloc.find(k); + + if (h) { + movedButStillInMemory++; + /* All moved elements should be in the second tier. */ + EXPECT_TRUE(alloc.allocator_[1]->isMemoryInAllocator(h->getMemory())); + EXPECT_EQ(data, std::string((char*)h->getMemory(), data.size())); + } + } + + EXPECT_TRUE(movedButStillInMemory > 0); + } }; } // namespace tests } // namespace cachelib From 94c4974b103cd186bf98646726e79b4ea252f784 Mon Sep 17 00:00:00 2001 From: Sergei Vinogradov Date: Thu, 27 Jan 2022 05:27:20 -0800 Subject: [PATCH 13/31] Aadding new configs to hit_ratio/graph_cache_leader_fobj -updated configs for numa bindings --- .../config-4GB-DRAM-4GB-PMEM.json | 42 +++++++++++++++++++ .../config-8GB-DRAM.json | 32 ++++++++++++++ .../config-8GB-PMEM.json | 38 +++++++++++++++++ .../test_configs/simple_tiers_test.json | 12 ++++-- 4 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-4GB-DRAM-4GB-PMEM.json create mode 100644 cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-8GB-DRAM.json create mode 100644 cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-8GB-PMEM.json diff --git a/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-4GB-DRAM-4GB-PMEM.json b/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-4GB-DRAM-4GB-PMEM.json new file mode 100644 index 0000000000..d9acdf7c6c --- /dev/null +++ b/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-4GB-DRAM-4GB-PMEM.json @@ -0,0 +1,42 @@ +{ + "cache_config": { + "cacheSizeMB": 8192, + "poolRebalanceIntervalSec": 0, + "cacheDir": "/tmp/mem-tiers", + "memoryTiers" : [ + { + "ratio": 1, + "memBindNodes": 0 + }, + { + "ratio": 1, + "memBindNodes": 0 + } + ] + }, + "test_config": + { + "addChainedRatio": 0.0, + "delRatio": 0.0, + "enableLookaside": true, + "getRatio": 0.7684563460126871, + "keySizeRange": [ + 1, + 8, + 64 + ], + "keySizeRangeProbability": [ + 0.3, + 0.7 + ], + "loneGetRatio": 0.2315436539873129, + "numKeys": 71605574, + "numOps": 5000000, + "numThreads": 24, + "popDistFile": "pop.json", + + "setRatio": 0.0, + "valSizeDistFile": "sizes.json" + } + +} diff --git a/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-8GB-DRAM.json b/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-8GB-DRAM.json new file mode 100644 index 0000000000..6d47e08b74 --- /dev/null +++ b/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-8GB-DRAM.json @@ -0,0 +1,32 @@ +{ + "cache_config": { + "cacheSizeMB": 8192, + "poolRebalanceIntervalSec": 0, + "cacheDir": "/tmp/mem-tier" + }, + "test_config": + { + "addChainedRatio": 0.0, + "delRatio": 0.0, + "enableLookaside": true, + "getRatio": 0.7684563460126871, + "keySizeRange": [ + 1, + 8, + 64 + ], + "keySizeRangeProbability": [ + 0.3, + 0.7 + ], + "loneGetRatio": 0.2315436539873129, + "numKeys": 71605574, + "numOps": 5000000, + "numThreads": 24, + "popDistFile": "pop.json", + + "setRatio": 0.0, + "valSizeDistFile": "sizes.json" + } + +} diff --git a/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-8GB-PMEM.json b/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-8GB-PMEM.json new file mode 100644 index 0000000000..4feab55154 --- /dev/null +++ b/cachelib/cachebench/test_configs/hit_ratio/graph_cache_leader_fbobj/config-8GB-PMEM.json @@ -0,0 +1,38 @@ +{ + "cache_config": { + "cacheSizeMB": 8192, + "poolRebalanceIntervalSec": 0, + "cacheDir": "/tmp/mem-tier", + "memoryTiers" : [ + { + "ratio": 1, + "memBindNodes": 0 + } + ] + }, + "test_config": + { + "addChainedRatio": 0.0, + "delRatio": 0.0, + "enableLookaside": true, + "getRatio": 0.7684563460126871, + "keySizeRange": [ + 1, + 8, + 64 + ], + "keySizeRangeProbability": [ + 0.3, + 0.7 + ], + "loneGetRatio": 0.2315436539873129, + "numKeys": 71605574, + "numOps": 5000000, + "numThreads": 24, + "popDistFile": "pop.json", + + "setRatio": 0.0, + "valSizeDistFile": "sizes.json" + } + +} diff --git a/cachelib/cachebench/test_configs/simple_tiers_test.json b/cachelib/cachebench/test_configs/simple_tiers_test.json index 182bb514cb..58302b9f20 100644 --- a/cachelib/cachebench/test_configs/simple_tiers_test.json +++ b/cachelib/cachebench/test_configs/simple_tiers_test.json @@ -1,14 +1,18 @@ // @nolint instantiates a small cache and runs a quick run of basic operations. { "cache_config" : { - "cacheSizeMB" : 512, - "usePosixShm" : false, + "cacheSizeMB" : 1024, "cacheDir" : "/tmp/mem-tiers", "memoryTiers" : [ + { + "ratio": 1, + "memBindNodes": "0" + }, { "ratio": 1, "memBindNodes": "0" } + ], "poolRebalanceIntervalSec" : 1, "moveOnSlabRelease" : false, @@ -19,7 +23,7 @@ "test_config" : { "numOps" : 100000, "numThreads" : 32, - "numKeys" : 1000000, + "numKeys" : 2000000, "keySizeRange" : [1, 8, 64], "keySizeRangeProbability" : [0.3, 0.7], @@ -33,4 +37,4 @@ "keyPoolDistribution": [0.4, 0.6], "opPoolDistribution" : [0.5, 0.5] } - } \ No newline at end of file + } From afd145650bfdae30b600c414488b0b181ef1b722 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Mon, 19 Dec 2022 08:43:35 -0800 Subject: [PATCH 14/31] Do not block reader if a child item is moving This would lead to deadlock (.e.g in forEachChainedItem) if the child is moving (e.g. marked by Slab Release thread). Instead treat moving bit only to prevent freeing the item and do all synchronization on parent. --- cachelib/allocator/CacheAllocator-inl.h | 93 +++++++++++++++---------- 1 file changed, 58 insertions(+), 35 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 47d4871f6b..60b72baa82 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -987,7 +987,8 @@ CacheAllocator::acquire(Item* it) { SCOPE_FAIL { stats_.numRefcountOverflow.inc(); }; - auto failIfMoving = getNumTiers() > 1; + // TODO: do not block incRef for child items to avoid deadlock + auto failIfMoving = getNumTiers() > 1 && !it->isChainedItem(); auto incRes = incRef(*it, failIfMoving); if (LIKELY(incRes == RefcountWithFlags::incResult::incOk)) { return WriteHandle{it, *this}; @@ -3051,7 +3052,8 @@ bool CacheAllocator::tryMovingForSlabRelease( // a regular item or chained item is synchronized with any potential // user-side mutation. std::unique_ptr syncObj; - if (config_.movingSync) { + if (config_.movingSync && getNumTiers() == 1) { + // TODO: use moving-bit synchronization for single tier as well if (!oldItem.isChainedItem()) { syncObj = config_.movingSync(oldItem.getKey()); } else { @@ -3149,47 +3151,51 @@ void CacheAllocator::evictForSlabRelease( Item* evicted; if (item.isChainedItem()) { auto& expectedParent = item.asChainedItem().getParentItem(compressor_); - const std::string parentKey = expectedParent.getKey().str(); - auto l = chainedItemLocks_.lockExclusive(parentKey); - - // check if the child is still in mmContainer and the expected parent is - // valid under the chained item lock. - if (expectedParent.getKey() != parentKey || !item.isInMMContainer() || - item.isOnlyMoving() || - &expectedParent != &item.asChainedItem().getParentItem(compressor_) || - !expectedParent.isAccessible() || !expectedParent.hasChainedItem()) { - continue; - } - // search if the child is present in the chain - { - auto parentHandle = findInternal(parentKey); - if (!parentHandle || parentHandle != &expectedParent) { + if (getNumTiers() == 1) { + // TODO: unify this with multi-tier implementation + // right now, taking a chained item lock here would lead to deadlock + const std::string parentKey = expectedParent.getKey().str(); + auto l = chainedItemLocks_.lockExclusive(parentKey); + + // check if the child is still in mmContainer and the expected parent is + // valid under the chained item lock. + if (expectedParent.getKey() != parentKey || !item.isInMMContainer() || + item.isOnlyMoving() || + &expectedParent != &item.asChainedItem().getParentItem(compressor_) || + !expectedParent.isAccessible() || !expectedParent.hasChainedItem()) { continue; } - ChainedItem* head = nullptr; - { // scope for the handle - auto headHandle = findChainedItem(expectedParent); - head = headHandle ? &headHandle->asChainedItem() : nullptr; - } + // search if the child is present in the chain + { + auto parentHandle = findInternal(parentKey); + if (!parentHandle || parentHandle != &expectedParent) { + continue; + } - bool found = false; - while (head) { - if (head == &item) { - found = true; - break; + ChainedItem* head = nullptr; + { // scope for the handle + auto headHandle = findChainedItem(expectedParent); + head = headHandle ? &headHandle->asChainedItem() : nullptr; } - head = head->getNext(compressor_); - } - if (!found) { - continue; + bool found = false; + while (head) { + if (head == &item) { + found = true; + break; + } + head = head->getNext(compressor_); + } + + if (!found) { + continue; + } } } evicted = &expectedParent; - token = createPutToken(*evicted); if (evicted->markForEviction()) { // unmark the child so it will be freed @@ -3200,6 +3206,9 @@ void CacheAllocator::evictForSlabRelease( // no other reader can be added to the waiters list wakeUpWaiters(*evicted, {}); } else { + // TODO: potential deadlock with markUseful for parent item + // for now, we do not block any reader on child items but this + // should probably be fixed continue; } } else { @@ -3231,7 +3240,17 @@ void CacheAllocator::evictForSlabRelease( XDCHECK(evicted->getRefCount() == 0); const auto res = releaseBackToAllocator(*evicted, RemoveContext::kEviction, false); - XDCHECK(res == ReleaseRes::kReleased); + + if (getNumTiers() == 1) { + XDCHECK(res == ReleaseRes::kReleased); + } else { + const bool isAlreadyFreed = + !markMovingForSlabRelease(ctx, &item, throttler); + if (!isAlreadyFreed) { + continue; + } + } + return; } } @@ -3279,11 +3298,15 @@ bool CacheAllocator::markMovingForSlabRelease( bool itemFreed = true; bool markedMoving = false; TierId tid = getTierId(alloc); - const auto fn = [&markedMoving, &itemFreed](void* memory) { + auto numTiers = getNumTiers(); + const auto fn = [&markedMoving, &itemFreed, numTiers](void* memory) { // Since this callback is executed, the item is not yet freed itemFreed = false; Item* item = static_cast(memory); - if (item->markMoving(false)) { + // TODO: for chained items, moving bit is only used to avoid + // freeing the item prematurely + auto failIfRefNotZero = numTiers > 1 && !item->isChainedItem(); + if (item->markMoving(failIfRefNotZero)) { markedMoving = true; } }; From 4f8f425f7f3a5242809589e05583a1d614768b70 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Fri, 21 Oct 2022 12:27:47 -0400 Subject: [PATCH 15/31] Background data movement (#20) Background data movement using periodic workers. Attempts to evict/promote items per given thresholds for each class. These reduce p99 latency since there is a higher chance that an allocation slot is free in the tier we are allocating in. fix race in promotion where releaseBackToAllocator was being called before wakeUpWaiters. reinsert to mm container on failed promotion --- MultiTierDataMovement.md | 90 +++++++ cachelib/allocator/BackgroundMover-inl.h | 112 ++++++++ cachelib/allocator/BackgroundMover.h | 103 ++++++++ cachelib/allocator/BackgroundMoverStrategy.h | 42 +++ cachelib/allocator/CMakeLists.txt | 1 + cachelib/allocator/Cache.h | 6 + cachelib/allocator/CacheAllocator-inl.h | 188 +++++++++++++- cachelib/allocator/CacheAllocator.h | 239 +++++++++++++++++- cachelib/allocator/CacheAllocatorConfig.h | 71 ++++++ cachelib/allocator/CacheStats.h | 25 ++ cachelib/allocator/FreeThresholdStrategy.cpp | 74 ++++++ cachelib/allocator/FreeThresholdStrategy.h | 46 ++++ cachelib/allocator/MMLru-inl.h | 22 +- cachelib/allocator/MMLru.h | 3 + cachelib/allocator/MMTinyLFU-inl.h | 7 + cachelib/allocator/MMTinyLFU.h | 3 + cachelib/allocator/PromotionStrategy.h | 84 ++++++ .../tests/AllocatorMemoryTiersTest.cpp | 1 + .../tests/AllocatorMemoryTiersTest.h | 67 +++++ cachelib/allocator/tests/CacheBaseTest.cpp | 2 + cachelib/cachebench/cache/Cache-inl.h | 34 +++ cachelib/cachebench/cache/CacheStats.h | 59 +++++ cachelib/cachebench/util/CacheConfig.cpp | 38 ++- cachelib/cachebench/util/CacheConfig.h | 27 ++ 24 files changed, 1318 insertions(+), 26 deletions(-) create mode 100644 MultiTierDataMovement.md create mode 100644 cachelib/allocator/BackgroundMover-inl.h create mode 100644 cachelib/allocator/BackgroundMover.h create mode 100644 cachelib/allocator/BackgroundMoverStrategy.h create mode 100644 cachelib/allocator/FreeThresholdStrategy.cpp create mode 100644 cachelib/allocator/FreeThresholdStrategy.h create mode 100644 cachelib/allocator/PromotionStrategy.h diff --git a/MultiTierDataMovement.md b/MultiTierDataMovement.md new file mode 100644 index 0000000000..cccc14b947 --- /dev/null +++ b/MultiTierDataMovement.md @@ -0,0 +1,90 @@ +# Background Data Movement + +In order to reduce the number of online evictions and support asynchronous +promotion - we have added two periodic workers to handle eviction and promotion. + +The diagram below shows a simplified version of how the background evictor +thread (green) is integrated to the CacheLib architecture. + +

+ BackgroundEvictor +

+ +## Background Evictors + +The background evictors scan each class to see if there are objects to move the next (lower) +tier using a given strategy. Here we document the parameters for the different +strategies and general parameters. + +- `backgroundEvictorIntervalMilSec`: The interval that this thread runs for - by default +the background evictor threads will wake up every 10 ms to scan the AllocationClasses. Also, +the background evictor thread will be woken up everytime there is a failed allocation (from +a request handling thread) and the current percentage of free memory for the +AllocationClass is lower than `lowEvictionAcWatermark`. This may render the interval parameter +not as important when there are many allocations occuring from request handling threads. + +- `evictorThreads`: The number of background evictors to run - each thread is a assigned +a set of AllocationClasses to scan and evict objects from. Currently, each thread gets +an equal number of classes to scan - but as object size distribution may be unequal - future +versions will attempt to balance the classes among threads. The range is 1 to number of AllocationClasses. +The default is 1. + +- `maxEvictionBatch`: The number of objects to remove in a given eviction call. The +default is 40. Lower range is 10 and the upper range is 1000. Too low and we might not +remove objects at a reasonable rate, too high and it might increase contention with user threads. + +- `minEvictionBatch`: Minimum number of items to evict at any time (if there are any +candidates) + +- `maxEvictionPromotionHotness`: Maximum candidates to consider for eviction. This is similar to `maxEvictionBatch` +but it specifies how many candidates will be taken into consideration, not the actual number of items to evict. +This option can be used to configure duration of critical section on LRU lock. + + +### FreeThresholdStrategy (default) + +- `lowEvictionAcWatermark`: Triggers background eviction thread to run +when this percentage of the AllocationClass is free. +The default is `2.0`, to avoid wasting capacity we don't set this above `10.0`. + +- `highEvictionAcWatermark`: Stop the evictions from an AllocationClass when this +percentage of the AllocationClass is free. The default is `5.0`, to avoid wasting capacity we +don't set this above `10`. + + +## Background Promoters + +The background promoters scan each class to see if there are objects to move to a lower +tier using a given strategy. Here we document the parameters for the different +strategies and general parameters. + +- `backgroundPromoterIntervalMilSec`: The interval that this thread runs for - by default +the background promoter threads will wake up every 10 ms to scan the AllocationClasses for +objects to promote. + +- `promoterThreads`: The number of background promoters to run - each thread is a assigned +a set of AllocationClasses to scan and promote objects from. Currently, each thread gets +an equal number of classes to scan - but as object size distribution may be unequal - future +versions will attempt to balance the classes among threads. The range is `1` to number of AllocationClasses. The default is `1`. + +- `maxProtmotionBatch`: The number of objects to promote in a given promotion call. The +default is 40. Lower range is 10 and the upper range is 1000. Too low and we might not +remove objects at a reasonable rate, too high and it might increase contention with user threads. + +- `minPromotionBatch`: Minimum number of items to promote at any time (if there are any +candidates) + +- `numDuplicateElements`: This allows us to promote items that have existing handles (read-only) since +we won't need to modify the data when a user is done with the data. Therefore, for a short time +the data could reside in both tiers until it is evicted from its current tier. The default is to +not allow this (0). Setting the value to 100 will enable duplicate elements in tiers. + +### Background Promotion Strategy (only one currently) + +- `promotionAcWatermark`: Promote items if there is at least this +percent of free AllocationClasses. Promotion thread will attempt to move `maxPromotionBatch` number of objects +to that tier. The objects are chosen from the head of the LRU. The default is `4.0`. +This value should correlate with `lowEvictionAcWatermark`, `highEvictionAcWatermark`, `minAcAllocationWatermark`, `maxAcAllocationWatermark`. +- `maxPromotionBatch`: The number of objects to promote in batch during BG promotion. Analogous to +`maxEvictionBatch`. It's value should be lower to decrease contention on hot items. + diff --git a/cachelib/allocator/BackgroundMover-inl.h b/cachelib/allocator/BackgroundMover-inl.h new file mode 100644 index 0000000000..b77436635f --- /dev/null +++ b/cachelib/allocator/BackgroundMover-inl.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace facebook { +namespace cachelib { + +template +BackgroundMover::BackgroundMover( + Cache& cache, + std::shared_ptr strategy, + MoverDir direction) + : cache_(cache), strategy_(strategy), direction_(direction) { + if (direction_ == MoverDir::Evict) { + moverFunc = BackgroundMoverAPIWrapper::traverseAndEvictItems; + + } else if (direction_ == MoverDir::Promote) { + moverFunc = BackgroundMoverAPIWrapper::traverseAndPromoteItems; + } +} + +template +BackgroundMover::~BackgroundMover() { + stop(std::chrono::seconds(0)); +} + +template +void BackgroundMover::work() { + try { + checkAndRun(); + } catch (const std::exception& ex) { + XLOGF(ERR, "BackgroundMover interrupted due to exception: {}", ex.what()); + } +} + +template +void BackgroundMover::setAssignedMemory( + std::vector&& assignedMemory) { + XLOG(INFO, "Class assigned to background worker:"); + for (auto [tid, pid, cid] : assignedMemory) { + XLOGF(INFO, "Tid: {}, Pid: {}, Cid: {}", tid, pid, cid); + } + + mutex.lock_combine([this, &assignedMemory] { + this->assignedMemory_ = std::move(assignedMemory); + }); +} + +// Look for classes that exceed the target memory capacity +// and return those for eviction +template +void BackgroundMover::checkAndRun() { + auto assignedMemory = mutex.lock_combine([this] { return assignedMemory_; }); + + unsigned int moves = 0; + std::set classes{}; + auto batches = strategy_->calculateBatchSizes(cache_, assignedMemory); + + for (size_t i = 0; i < batches.size(); i++) { + const auto [tid, pid, cid] = assignedMemory[i]; + const auto batch = batches[i]; + + classes.insert(cid); + const auto& mpStats = cache_.getPoolByTid(pid, tid).getStats(); + + if (!batch) { + continue; + } + + // try moving BATCH items from the class in order to reach free target + auto moved = moverFunc(cache_, tid, pid, cid, batch); + moves += moved; + moves_per_class_[tid][pid][cid] += moved; + totalBytesMoved.add(moved * mpStats.acStats.at(cid).allocSize); + } + + numTraversals.inc(); + numMovedItems.add(moves); + totalClasses.add(classes.size()); +} + +template +BackgroundMoverStats BackgroundMover::getStats() const noexcept { + BackgroundMoverStats stats; + stats.numMovedItems = numMovedItems.get(); + stats.runCount = numTraversals.get(); + stats.totalBytesMoved = totalBytesMoved.get(); + stats.totalClasses = totalClasses.get(); + + return stats; +} + +template +std::map>> +BackgroundMover::getClassStats() const noexcept { + return moves_per_class_; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/BackgroundMover.h b/cachelib/allocator/BackgroundMover.h new file mode 100644 index 0000000000..1246676d6e --- /dev/null +++ b/cachelib/allocator/BackgroundMover.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/BackgroundMoverStrategy.h" +#include "cachelib/allocator/CacheStats.h" +#include "cachelib/common/AtomicCounter.h" +#include "cachelib/common/PeriodicWorker.h" + +namespace facebook { +namespace cachelib { + +// wrapper that exposes the private APIs of CacheType that are specifically +// needed for the cache api +template +struct BackgroundMoverAPIWrapper { + static size_t traverseAndEvictItems(C& cache, + unsigned int tid, + unsigned int pid, + unsigned int cid, + size_t batch) { + return cache.traverseAndEvictItems(tid, pid, cid, batch); + } + + static size_t traverseAndPromoteItems(C& cache, + unsigned int tid, + unsigned int pid, + unsigned int cid, + size_t batch) { + return cache.traverseAndPromoteItems(tid, pid, cid, batch); + } +}; + +enum class MoverDir { Evict = 0, Promote }; + +// Periodic worker that evicts items from tiers in batches +// The primary aim is to reduce insertion times for new items in the +// cache +template +class BackgroundMover : public PeriodicWorker { + public: + using Cache = CacheT; + // @param cache the cache interface + // @param strategy the stragey class that defines how objects are + // moved, + // (promoted vs. evicted and how much) + BackgroundMover(Cache& cache, + std::shared_ptr strategy, + MoverDir direction_); + + ~BackgroundMover() override; + + BackgroundMoverStats getStats() const noexcept; + std::map>> + getClassStats() const noexcept; + + void setAssignedMemory( + std::vector&& assignedMemory); + + private: + std::map>> + moves_per_class_; + // cache allocator's interface for evicting + using Item = typename Cache::Item; + + Cache& cache_; + std::shared_ptr strategy_; + MoverDir direction_; + + std::function + moverFunc; + + // implements the actual logic of running the background evictor + void work() override final; + void checkAndRun(); + + AtomicCounter numMovedItems{0}; + AtomicCounter numTraversals{0}; + AtomicCounter totalClasses{0}; + AtomicCounter totalBytesMoved{0}; + + std::vector assignedMemory_; + folly::DistributedMutex mutex; +}; +} // namespace cachelib +} // namespace facebook + +#include "cachelib/allocator/BackgroundMover-inl.h" diff --git a/cachelib/allocator/BackgroundMoverStrategy.h b/cachelib/allocator/BackgroundMoverStrategy.h new file mode 100644 index 0000000000..7706a625a5 --- /dev/null +++ b/cachelib/allocator/BackgroundMoverStrategy.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/Cache.h" + + +namespace facebook { +namespace cachelib { + +struct MemoryDescriptorType { + MemoryDescriptorType(TierId tid, PoolId pid, ClassId cid) : + tid_(tid), pid_(pid), cid_(cid) {} + TierId tid_; + PoolId pid_; + ClassId cid_; +}; + +// Base class for background eviction strategy. +class BackgroundMoverStrategy { + public: + virtual std::vector calculateBatchSizes( + const CacheBase& cache, + std::vector acVec) = 0; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/CMakeLists.txt b/cachelib/allocator/CMakeLists.txt index f94c8c90c7..6103cdc823 100644 --- a/cachelib/allocator/CMakeLists.txt +++ b/cachelib/allocator/CMakeLists.txt @@ -35,6 +35,7 @@ add_library (cachelib_allocator CCacheManager.cpp ContainerTypes.cpp FreeMemStrategy.cpp + FreeThresholdStrategy.cpp HitsPerSlabStrategy.cpp LruTailAgeStrategy.cpp MarginalHitsOptimizeStrategy.cpp diff --git a/cachelib/allocator/Cache.h b/cachelib/allocator/Cache.h index cb2fa83f0d..c871358189 100644 --- a/cachelib/allocator/Cache.h +++ b/cachelib/allocator/Cache.h @@ -98,6 +98,12 @@ class CacheBase { // // @param poolId The pool id to query virtual const MemoryPool& getPool(PoolId poolId) const = 0; + + // Get the reference to a memory pool using a tier id, for stats purposes + // + // @param poolId The pool id to query + // @param tierId The tier of the pool id + virtual const MemoryPool& getPoolByTid(PoolId poolId, TierId tid) const = 0; // Get Pool specific stats (regular pools). This includes stats from the // Memory Pool and also the cache. diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 60b72baa82..47ba92eb94 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -290,6 +290,18 @@ void CacheAllocator::initWorkers() { config_.poolOptimizeStrategy, config_.ccacheOptimizeStepSizePercent); } + + if (config_.backgroundEvictorEnabled()) { + startNewBackgroundEvictor(config_.backgroundEvictorInterval, + config_.backgroundEvictorStrategy, + config_.backgroundEvictorThreads); + } + + if (config_.backgroundPromoterEnabled()) { + startNewBackgroundPromoter(config_.backgroundPromoterInterval, + config_.backgroundPromoterStrategy, + config_.backgroundPromoterThreads); + } } template @@ -368,6 +380,22 @@ CacheAllocator::allocate(PoolId poolId, ttlSecs == 0 ? 0 : creationTime + ttlSecs); } +template +bool CacheAllocator::shouldWakeupBgEvictor(TierId tid, PoolId pid, ClassId cid) { + // TODO: should we also work on lower tiers? should we have separate set of params? + if (tid == 1) return false; + return (1-getACStats(tid, pid, cid).usageFraction())*100 <= config_.lowEvictionAcWatermark; +} + +template +size_t CacheAllocator::backgroundWorkerId(TierId tid, PoolId pid, ClassId cid, size_t numWorkers) { + XDCHECK(numWorkers); + + // TODO: came up with some better sharding (use some hashing) + return (tid + pid + cid) % numWorkers; +} + + template typename CacheAllocator::WriteHandle CacheAllocator::allocateInternalTier(TierId tid, @@ -375,7 +403,8 @@ CacheAllocator::allocateInternalTier(TierId tid, typename Item::Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime) { + uint32_t expiryTime, + bool fromBgThread) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -390,8 +419,13 @@ CacheAllocator::allocateInternalTier(TierId tid, // TODO: per-tier (*stats_.allocAttempts)[pid][cid].inc(); - + void* memory = allocator_[tid]->allocate(pid, requiredSize); + + if (backgroundEvictor_.size() && !fromBgThread && (memory == nullptr || shouldWakeupBgEvictor(tid, pid, cid))) { + backgroundEvictor_[backgroundWorkerId(tid, pid, cid, backgroundEvictor_.size())]->wakeUp(); + } + if (memory == nullptr) { memory = findEviction(tid, pid, cid); } @@ -438,10 +472,11 @@ CacheAllocator::allocateInternal(PoolId pid, typename Item::Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime) { + uint32_t expiryTime, + bool fromBgThread) { auto tid = 0; /* TODO: consult admission policy */ for(TierId tid = 0; tid < getNumTiers(); ++tid) { - auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime); + auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime, fromBgThread); if (handle) return handle; } return {}; @@ -1589,7 +1624,7 @@ CacheAllocator::getNextCandidate(TierId tid, XDCHECK(candidate->isMoving() || candidate->isMarkedForEviction()); auto evictedToNext = lastTier ? nullptr - : tryEvictToNextMemoryTier(*candidate); + : tryEvictToNextMemoryTier(*candidate, false); if (!evictedToNext) { if (!token.isValid()) { token = createPutToken(*candidate); @@ -1719,7 +1754,7 @@ bool CacheAllocator::shouldWriteToNvmCacheExclusive( template typename CacheAllocator::WriteHandle CacheAllocator::tryEvictToNextMemoryTier( - TierId tid, PoolId pid, Item& item) { + TierId tid, PoolId pid, Item& item, bool fromBgThread) { XDCHECK(item.isMoving()); XDCHECK(item.getRefCount() == 0); if(item.hasChainedItem()) return WriteHandle{}; // TODO: We do not support ChainedItem yet @@ -1736,7 +1771,8 @@ CacheAllocator::tryEvictToNextMemoryTier( item.getKey(), item.getSize(), item.getCreationTime(), - item.getExpiryTime()); + item.getExpiryTime(), + fromBgThread); if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); @@ -1753,10 +1789,49 @@ CacheAllocator::tryEvictToNextMemoryTier( template typename CacheAllocator::WriteHandle -CacheAllocator::tryEvictToNextMemoryTier(Item& item) { +CacheAllocator::tryEvictToNextMemoryTier(Item& item, bool fromBgThread) { auto tid = getTierId(item); auto pid = allocator_[tid]->getAllocInfo(item.getMemory()).poolId; - return tryEvictToNextMemoryTier(tid, pid, item); + return tryEvictToNextMemoryTier(tid, pid, item, fromBgThread); +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::tryPromoteToNextMemoryTier( + TierId tid, PoolId pid, Item& item, bool fromBgThread) { + if(item.isExpired()) { return {}; } + TierId nextTier = tid; + while (nextTier > 0) { // try to evict down to the next memory tiers + auto toPromoteTier = nextTier - 1; + --nextTier; + + // allocateInternal might trigger another eviction + auto newItemHdl = allocateInternalTier(toPromoteTier, pid, + item.getKey(), + item.getSize(), + item.getCreationTime(), + item.getExpiryTime(), + fromBgThread); + + if (newItemHdl) { + XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); + moveRegularItemWithSync(item, newItemHdl); + item.unmarkMoving(); + return newItemHdl; + } else { + return WriteHandle{}; + } + } + + return {}; +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::tryPromoteToNextMemoryTier(Item& item, bool fromBgThread) { + auto tid = getTierId(item); + auto pid = allocator_[tid]->getAllocInfo(item.getMemory()).poolId; + return tryPromoteToNextMemoryTier(tid, pid, item, fromBgThread); } template @@ -2499,6 +2574,16 @@ PoolId CacheAllocator::addPool( setRebalanceStrategy(pid, std::move(rebalanceStrategy)); setResizeStrategy(pid, std::move(resizeStrategy)); + if (backgroundEvictor_.size()) { + for (size_t id = 0; id < backgroundEvictor_.size(); id++) + backgroundEvictor_[id]->setAssignedMemory(getAssignedMemoryToBgWorker(id, backgroundEvictor_.size(), 0)); + } + + if (backgroundPromoter_.size()) { + for (size_t id = 0; id < backgroundPromoter_.size(); id++) + backgroundPromoter_[id]->setAssignedMemory(getAssignedMemoryToBgWorker(id, backgroundPromoter_.size(), 1)); + } + return pid; } @@ -3033,7 +3118,8 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { oldItem.getKey(), oldItem.getSize(), oldItem.getCreationTime(), - oldItem.getExpiryTime()); + oldItem.getExpiryTime(), + false); if (!newItemHdl) { return {}; } @@ -3512,6 +3598,8 @@ bool CacheAllocator::stopWorkers(std::chrono::seconds timeout) { success &= stopPoolResizer(timeout); success &= stopMemMonitor(timeout); success &= stopReaper(timeout); + success &= stopBackgroundEvictor(timeout); + success &= stopBackgroundPromoter(timeout); return success; } @@ -3774,6 +3862,8 @@ GlobalCacheStats CacheAllocator::getGlobalCacheStats() const { ret.nvmCacheEnabled = nvmCache_ ? nvmCache_->isEnabled() : false; ret.reaperStats = getReaperStats(); ret.rebalancerStats = getRebalancerStats(); + ret.evictionStats = getBackgroundMoverStats(MoverDir::Evict); + ret.promotionStats = getBackgroundMoverStats(MoverDir::Promote); ret.numActiveHandles = getNumActiveHandles(); ret.isNewRamCache = cacheCreationTime_ == cacheInstanceCreationTime_; @@ -3944,6 +4034,64 @@ bool CacheAllocator::startNewReaper( return true; } +template +auto CacheAllocator::getAssignedMemoryToBgWorker(size_t evictorId, size_t numWorkers, TierId tid) +{ + std::vector asssignedMemory; + // TODO: for now, only evict from tier 0 + auto pools = filterCompactCachePools(allocator_[tid]->getPoolIds()); + for (const auto pid : pools) { + const auto& mpStats = getPoolByTid(pid,tid).getStats(); + for (const auto cid : mpStats.classIds) { + if (backgroundWorkerId(tid, pid, cid, numWorkers) == evictorId) { + asssignedMemory.emplace_back(tid, pid, cid); + } + } + } + return asssignedMemory; +} + +template +bool CacheAllocator::startNewBackgroundEvictor( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads) { + XDCHECK(threads > 0); + backgroundEvictor_.resize(threads); + bool result = true; + + for (size_t i = 0; i < threads; i++) { + auto ret = startNewWorker("BackgroundEvictor" + std::to_string(i), backgroundEvictor_[i], interval, *this, strategy, MoverDir::Evict); + result = result && ret; + + if (result) { + backgroundEvictor_[i]->setAssignedMemory(getAssignedMemoryToBgWorker(i, backgroundEvictor_.size(), 0)); + } + } + return result; +} + +template +bool CacheAllocator::startNewBackgroundPromoter( + std::chrono::milliseconds interval, + std::shared_ptr strategy, + size_t threads) { + XDCHECK(threads > 0); + XDCHECK(getNumTiers() > 1); + backgroundPromoter_.resize(threads); + bool result = true; + + for (size_t i = 0; i < threads; i++) { + auto ret = startNewWorker("BackgroundPromoter" + std::to_string(i), backgroundPromoter_[i], interval, *this, strategy, MoverDir::Promote); + result = result && ret; + + if (result) { + backgroundPromoter_[i]->setAssignedMemory(getAssignedMemoryToBgWorker(i, backgroundPromoter_.size(), 1)); + } + } + return result; +} + template bool CacheAllocator::stopPoolRebalancer( std::chrono::seconds timeout) { @@ -3992,6 +4140,26 @@ bool CacheAllocator::stopReaper(std::chrono::seconds timeout) { return res; } +template +bool CacheAllocator::stopBackgroundEvictor(std::chrono::seconds timeout) { + bool result = true; + for (size_t i = 0; i < backgroundEvictor_.size(); i++) { + auto ret = stopWorker("BackgroundEvictor", backgroundEvictor_[i], timeout); + result = result && ret; + } + return result; +} + +template +bool CacheAllocator::stopBackgroundPromoter(std::chrono::seconds timeout) { + bool result = true; + for (size_t i = 0; i < backgroundPromoter_.size(); i++) { + auto ret = stopWorker("BackgroundPromoter", backgroundPromoter_[i], timeout); + result = result && ret; + } + return result; +} + template bool CacheAllocator::cleanupStrayShmSegments( const std::string& cacheDir, bool posix) { diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index bc228d1c5b..597e81f577 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -40,6 +40,7 @@ #include #pragma GCC diagnostic pop +#include "cachelib/allocator/BackgroundMover.h" #include "cachelib/allocator/CCacheManager.h" #include "cachelib/allocator/Cache.h" #include "cachelib/allocator/CacheAllocatorConfig.h" @@ -712,6 +713,11 @@ class CacheAllocator : public CacheBase { // @return the full usable size for this item uint32_t getUsableSize(const Item& item) const; + // gets the allocation class assigned to BG worker + auto getAssignedMemoryToBgWorker(size_t evictorId, size_t numWorkers, TierId tid); + bool shouldWakeupBgEvictor(TierId tid, PoolId pid, ClassId cid); + size_t backgroundWorkerId(TierId tid, PoolId pid, ClassId cid, size_t numWorkers); + // Get a random item from memory // This is useful for profiling and sampling cachelib managed memory // @@ -1057,6 +1063,11 @@ class CacheAllocator : public CacheBase { // @param reaperThrottleConfig throttling config bool startNewReaper(std::chrono::milliseconds interval, util::Throttler::Config reaperThrottleConfig); + + bool startNewBackgroundPromoter(std::chrono::milliseconds interval, + std::shared_ptr strategy, size_t threads); + bool startNewBackgroundEvictor(std::chrono::milliseconds interval, + std::shared_ptr strategy, size_t threads); // Stop existing workers with a timeout bool stopPoolRebalancer(std::chrono::seconds timeout = std::chrono::seconds{ @@ -1066,6 +1077,8 @@ class CacheAllocator : public CacheBase { 0}); bool stopMemMonitor(std::chrono::seconds timeout = std::chrono::seconds{0}); bool stopReaper(std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundEvictor(std::chrono::seconds timeout = std::chrono::seconds{0}); + bool stopBackgroundPromoter(std::chrono::seconds timeout = std::chrono::seconds{0}); // Set pool optimization to either true or false // @@ -1101,6 +1114,10 @@ class CacheAllocator : public CacheBase { return allocator_[currentTier()]->getPool(pid); } + const MemoryPool& getPoolByTid(PoolId pid, TierId tid) const override final { + return allocator_[tid]->getPool(pid); + } + // calculate the number of slabs to be advised/reclaimed in each pool PoolAdviseReclaimData calcNumSlabsToAdviseReclaim() override final { auto regularPoolIds = getRegularPoolIds(); @@ -1151,6 +1168,52 @@ class CacheAllocator : public CacheBase { auto stats = reaper_ ? reaper_->getStats() : ReaperStats{}; return stats; } + + // returns the background mover stats + BackgroundMoverStats getBackgroundMoverStats(MoverDir direction) const { + + auto stats = BackgroundMoverStats{}; + if (direction == MoverDir::Evict) { + for (auto &bg : backgroundEvictor_) + stats += bg->getStats(); + } else if (direction == MoverDir::Promote) { + for (auto &bg : backgroundPromoter_) + stats += bg->getStats(); + } + return stats; + + } + + + std::map>> + getBackgroundMoverClassStats(MoverDir direction) const { + std::map>> stats; + + if (direction == MoverDir::Evict) { + for (auto &bg : backgroundEvictor_) { + for (auto &tid : bg->getClassStats()) { + for (auto &pid : tid.second) { + for (auto &cid : pid.second) { + stats[tid.first][pid.first][cid.first] += cid.second; + } + } + } + } + } else if (direction == MoverDir::Promote) { + for (auto &bg : backgroundPromoter_) { + for (auto &tid : bg->getClassStats()) { + for (auto &pid : tid.second) { + for (auto &cid : pid.second) { + stats[tid.first][pid.first][cid.first] += cid.second; + } + } + } + } + } + + return stats; + } + // returns the pool rebalancer stats RebalancerStats getRebalancerStats() const { @@ -1448,7 +1511,8 @@ class CacheAllocator : public CacheBase { Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime); + uint32_t expiryTime, + bool fromBgThread = false); // create a new cache allocation on specific memory tier. // For description see allocateInternal. @@ -1459,7 +1523,8 @@ class CacheAllocator : public CacheBase { Key key, uint32_t size, uint32_t creationTime, - uint32_t expiryTime); + uint32_t expiryTime, + bool fromBgThread); // Allocate a chained item // @@ -1751,7 +1816,11 @@ class CacheAllocator : public CacheBase { // // @return valid handle to the item. This will be the last // handle to the item. On failure an empty handle. - WriteHandle tryEvictToNextMemoryTier(TierId tid, PoolId pid, Item& item); + WriteHandle tryEvictToNextMemoryTier(TierId tid, PoolId pid, Item& item, bool fromBgThread); + + WriteHandle tryPromoteToNextMemoryTier(TierId tid, PoolId pid, Item& item, bool fromBgThread); + + WriteHandle tryPromoteToNextMemoryTier(Item& item, bool fromBgThread); // Wakes up waiters if there are any // @@ -1771,7 +1840,7 @@ class CacheAllocator : public CacheBase { // // @return valid handle to the item. This will be the last // handle to the item. On failure an empty handle. - WriteHandle tryEvictToNextMemoryTier(Item& item); + WriteHandle tryEvictToNextMemoryTier(Item& item, bool fromBgThread); // Deserializer CacheAllocatorMetadata and verify the version // @@ -1916,6 +1985,163 @@ class CacheAllocator : public CacheBase { stats().numReaperSkippedSlabs.add(slabsSkipped); } + // exposed for the background evictor to iterate through the memory and evict + // in batch. This should improve insertion path for tiered memory config + size_t traverseAndEvictItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { +auto& mmContainer = getMMContainer(tid, pid, cid); + size_t evictions = 0; + size_t evictionCandidates = 0; + std::vector candidates; + candidates.reserve(batch); + + size_t tries = 0; + mmContainer.withEvictionIterator([&tries, &candidates, &batch, &mmContainer, this](auto &&itr) { + while (candidates.size() < batch && + (config_.maxEvictionPromotionHotness == 0 || tries < config_.maxEvictionPromotionHotness) && + itr) { + tries++; + Item* candidate = itr.get(); + XDCHECK(candidate); + + if (candidate->isChainedItem()) { + throw std::runtime_error("Not supported for chained items"); + } + + if (candidate->markMoving(true)) { + mmContainer.remove(itr); + candidates.push_back(candidate); + } else { + ++itr; + } + } + }); + + for (Item *candidate : candidates) { + auto evictedToNext = tryEvictToNextMemoryTier(*candidate, true /* from BgThread */); + if (!evictedToNext) { + auto token = createPutToken(*candidate); + + auto ret = candidate->markForEvictionWhenMoving(); + XDCHECK(ret); + + unlinkItemForEviction(*candidate); + // wake up any readers that wait for the move to complete + // it's safe to do now, as we have the item marked exclusive and + // no other reader can be added to the waiters list + wakeUpWaiters(*candidate, WriteHandle{}); + + if (token.isValid() && shouldWriteToNvmCacheExclusive(*candidate)) { + nvmCache_->put(*candidate, std::move(token)); + } + } else { + evictions++; + XDCHECK(!evictedToNext->isMarkedForEviction() && !evictedToNext->isMoving()); + XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); + XDCHECK(!candidate->isAccessible()); + XDCHECK(candidate->getKey() == evictedToNext->getKey()); + + wakeUpWaiters(*candidate, std::move(evictedToNext)); + } + XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); + + if (candidate->hasChainedItem()) { + (*stats_.chainedItemEvictions)[pid][cid].inc(); + } else { + (*stats_.regularItemEvictions)[pid][cid].inc(); + } + + // it's safe to recycle the item here as there are no more + // references and the item could not been marked as moving + // by other thread since it's detached from MMContainer. + auto res = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false); + XDCHECK(res == ReleaseRes::kReleased); + } + return evictions; + } + + size_t traverseAndPromoteItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + auto& mmContainer = getMMContainer(tid, pid, cid); + size_t promotions = 0; + std::vector candidates; + candidates.reserve(batch); + + size_t tries = 0; + + mmContainer.withPromotionIterator([&tries, &candidates, &batch, &mmContainer, this](auto &&itr){ + while (candidates.size() < batch && (config_.maxEvictionPromotionHotness == 0 || tries < config_.maxEvictionPromotionHotness) && itr) { + tries++; + Item* candidate = itr.get(); + XDCHECK(candidate); + + if (candidate->isChainedItem()) { + throw std::runtime_error("Not supported for chained items"); + } + + // TODO: only allow it for read-only items? + // or implement mvcc + if (candidate->markMoving(true)) { + // promotions should rarely fail since we already marked moving + mmContainer.remove(itr); + candidates.push_back(candidate); + } + + ++itr; + } + }); + + for (Item *candidate : candidates) { + auto promoted = tryPromoteToNextMemoryTier(*candidate, true); + if (promoted) { + promotions++; + XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); + // it's safe to recycle the item here as there are no more + // references and the item could not been marked as moving + // by other thread since it's detached from MMContainer. + // + // but we need to wake up waiters before releasing + // since candidate's key can change after being sent + // back to allocator + wakeUpWaiters(*candidate, std::move(promoted)); + auto res = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false); + XDCHECK(res == ReleaseRes::kReleased); + } else { + // we failed to allocate a new item, this item is no longer moving + auto ref = candidate->unmarkMoving(); + if (UNLIKELY(ref == 0)) { + wakeUpWaiters(*candidate,{}); + const auto res = + releaseBackToAllocator(*candidate, + RemoveContext::kNormal, false); + XDCHECK(res == ReleaseRes::kReleased); + } else if (candidate->isAccessible()) { + //case where we failed to allocate in lower tier + //item is still present in accessContainer + //item is no longer moving - acquire and + //wake up waiters with this handle + auto hdl = acquire(candidate); + insertInMMContainer(*hdl); + wakeUpWaiters(*candidate,std::move(hdl)); + } else if (!candidate->isAccessible()) { + //case where we failed to replace in access + //container due to another thread calling insertOrReplace + //unmark moving and return null handle + wakeUpWaiters(*candidate,{}); + if (UNLIKELY(ref == 0)) { + const auto res = + releaseBackToAllocator(*candidate, RemoveContext::kNormal, + false); + XDCHECK(res == ReleaseRes::kReleased); + } + } else { + XDCHECK(false); + } + } + } + return promotions; + } + // returns true if nvmcache is enabled and we should write this item to // nvmcache. bool shouldWriteToNvmCache(const Item& item); @@ -2239,6 +2465,10 @@ class CacheAllocator : public CacheBase { // free memory monitor std::unique_ptr memMonitor_; + + // background evictor + std::vector>> backgroundEvictor_; + std::vector>> backgroundPromoter_; // check whether a pool is a slabs pool std::array isCompactCachePool_{}; @@ -2300,6 +2530,7 @@ class CacheAllocator : public CacheBase { // Make this friend to give access to acquire and release friend ReadHandle; friend ReaperAPIWrapper; + friend BackgroundMoverAPIWrapper; friend class CacheAPIWrapperForNvm; friend class FbInternalRuntimeUpdateWrapper; friend class objcache2::ObjectCache; diff --git a/cachelib/allocator/CacheAllocatorConfig.h b/cachelib/allocator/CacheAllocatorConfig.h index 74a86e9789..925b07706b 100644 --- a/cachelib/allocator/CacheAllocatorConfig.h +++ b/cachelib/allocator/CacheAllocatorConfig.h @@ -31,6 +31,7 @@ #include "cachelib/allocator/MemoryTierCacheConfig.h" #include "cachelib/allocator/NvmAdmissionPolicy.h" #include "cachelib/allocator/PoolOptimizeStrategy.h" +#include "cachelib/allocator/BackgroundMoverStrategy.h" #include "cachelib/allocator/RebalanceStrategy.h" #include "cachelib/allocator/Util.h" #include "cachelib/common/EventInterface.h" @@ -267,6 +268,16 @@ class CacheAllocatorConfig { std::chrono::seconds regularInterval, std::chrono::seconds ccacheInterval, uint32_t ccacheStepSizePercent); + + // Enable the background evictor - scans a tier to look for objects + // to evict to the next tier + CacheAllocatorConfig& enableBackgroundEvictor( + std::shared_ptr backgroundMoverStrategy, + std::chrono::milliseconds regularInterval, size_t threads); + + CacheAllocatorConfig& enableBackgroundPromoter( + std::shared_ptr backgroundMoverStrategy, + std::chrono::milliseconds regularInterval, size_t threads); // This enables an optimization for Pool rebalancing and resizing. // The rough idea is to ensure only the least useful items are evicted when @@ -342,6 +353,17 @@ class CacheAllocatorConfig { poolOptimizeStrategy != nullptr; } + // @return whether background evictor thread is enabled + bool backgroundEvictorEnabled() const noexcept { + return backgroundEvictorInterval.count() > 0 && + backgroundEvictorStrategy != nullptr; + } + + bool backgroundPromoterEnabled() const noexcept { + return backgroundPromoterInterval.count() > 0 && + backgroundPromoterStrategy != nullptr; + } + // @return whether memory monitor is enabled bool memMonitoringEnabled() const noexcept { return memMonitorConfig.mode != MemoryMonitor::Disabled && @@ -455,6 +477,16 @@ class CacheAllocatorConfig { // The slab release process is considered as being stuck if it does not // make any progress for the below threshold std::chrono::milliseconds slabReleaseStuckThreshold{std::chrono::seconds(60)}; + + // rebalance to avoid alloc fialures. + std::shared_ptr backgroundEvictorStrategy; + std::shared_ptr backgroundPromoterStrategy; + // time interval to sleep between runs of the background evictor + std::chrono::milliseconds backgroundEvictorInterval{std::chrono::milliseconds{1000}}; + std::chrono::milliseconds backgroundPromoterInterval{std::chrono::milliseconds{1000}}; + + size_t backgroundEvictorThreads{1}; + size_t backgroundPromoterThreads{1}; // time interval to sleep between iterations of pool size optimization, // for regular pools and compact caches @@ -594,6 +626,25 @@ class CacheAllocatorConfig { // If true, we will delay worker start until user explicitly calls // CacheAllocator::startCacheWorkers() bool delayCacheWorkersStart{false}; + + // see MultiTierDataMovement.md + double promotionAcWatermark{4.0}; + double lowEvictionAcWatermark{2.0}; + double highEvictionAcWatermark{5.0}; + double numDuplicateElements{0.0}; // inclusivness of the cache + double syncPromotion{0.0}; // can promotion be done synchronously in user thread + + uint64_t evictorThreads{1}; + uint64_t promoterThreads{1}; + + uint64_t maxEvictionBatch{40}; + uint64_t maxPromotionBatch{10}; + + uint64_t minEvictionBatch{1}; + uint64_t minPromotionBatch{1}; + + uint64_t maxEvictionPromotionHotness{60}; + friend CacheT; @@ -933,6 +984,26 @@ CacheAllocatorConfig& CacheAllocatorConfig::enablePoolRebalancing( return *this; } +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundEvictor( + std::shared_ptr strategy, + std::chrono::milliseconds interval, size_t evictorThreads) { + backgroundEvictorStrategy = strategy; + backgroundEvictorInterval = interval; + backgroundEvictorThreads = evictorThreads; + return *this; +} + +template +CacheAllocatorConfig& CacheAllocatorConfig::enableBackgroundPromoter( + std::shared_ptr strategy, + std::chrono::milliseconds interval, size_t promoterThreads) { + backgroundPromoterStrategy = strategy; + backgroundPromoterInterval = interval; + backgroundPromoterThreads = promoterThreads; + return *this; +} + template CacheAllocatorConfig& CacheAllocatorConfig::enablePoolResizing( std::shared_ptr resizeStrategy, diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index 5de9ced173..11b5cf7596 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -301,6 +301,26 @@ struct RebalancerStats { uint64_t avgPickTimeMs{0}; }; +// Mover Stats +struct BackgroundMoverStats { + // the number of items this worker moved by looking at pools/classes stats + uint64_t numMovedItems{0}; + // number of times we went executed the thread //TODO: is this def correct? + uint64_t runCount{0}; + // total number of classes + uint64_t totalClasses{0}; + // eviction size + uint64_t totalBytesMoved{0}; + + BackgroundMoverStats& operator+=(const BackgroundMoverStats& rhs) { + numMovedItems += rhs.numMovedItems; + runCount += rhs.runCount; + totalClasses += rhs.totalClasses; + totalBytesMoved += rhs.totalBytesMoved; + return *this; + } +}; + // CacheMetadata type to export struct CacheMetadata { // allocator_version @@ -321,6 +341,11 @@ struct Stats; // Stats that apply globally in cache and // the ones that are aggregated over all pools struct GlobalCacheStats { + // background eviction stats + BackgroundMoverStats evictionStats; + + BackgroundMoverStats promotionStats; + // number of calls to CacheAllocator::find uint64_t numCacheGets{0}; diff --git a/cachelib/allocator/FreeThresholdStrategy.cpp b/cachelib/allocator/FreeThresholdStrategy.cpp new file mode 100644 index 0000000000..4a900c2cb1 --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) Intel and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cachelib/allocator/FreeThresholdStrategy.h" + +#include + +namespace facebook { +namespace cachelib { + +FreeThresholdStrategy::FreeThresholdStrategy(double lowEvictionAcWatermark, + double highEvictionAcWatermark, + uint64_t maxEvictionBatch, + uint64_t minEvictionBatch) + : lowEvictionAcWatermark(lowEvictionAcWatermark), + highEvictionAcWatermark(highEvictionAcWatermark), + maxEvictionBatch(maxEvictionBatch), + minEvictionBatch(minEvictionBatch) {} + +std::vector FreeThresholdStrategy::calculateBatchSizes( + const CacheBase& cache, + std::vector acVec) { + std::vector batches{}; + for (auto [tid, pid, cid] : acVec) { + auto stats = cache.getACStats(tid, pid, cid); + if ((1-stats.usageFraction())*100 >= highEvictionAcWatermark) { + batches.push_back(0); + } else { + auto toFreeMemPercent = highEvictionAcWatermark - (1-stats.usageFraction())*100; + auto toFreeItems = static_cast( + toFreeMemPercent * (stats.totalSlabs() * Slab::kSize) / stats.allocSize); + batches.push_back(toFreeItems); + } + } + + if (batches.size() == 0) { + return batches; + } + + auto maxBatch = *std::max_element(batches.begin(), batches.end()); + if (maxBatch == 0) + return batches; + + std::transform( + batches.begin(), batches.end(), batches.begin(), [&](auto numItems) { + if (numItems == 0) { + return 0UL; + } + + auto cappedBatchSize = maxEvictionBatch * numItems / maxBatch; + if (cappedBatchSize < minEvictionBatch) + return minEvictionBatch; + else + return cappedBatchSize; + }); + + return batches; +} + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/FreeThresholdStrategy.h b/cachelib/allocator/FreeThresholdStrategy.h new file mode 100644 index 0000000000..94316bfe82 --- /dev/null +++ b/cachelib/allocator/FreeThresholdStrategy.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/BackgroundMoverStrategy.h" +#include "cachelib/allocator/Cache.h" + +namespace facebook { +namespace cachelib { + +// Base class for background mover strategy. +class FreeThresholdStrategy : public BackgroundMoverStrategy { + public: + FreeThresholdStrategy(double lowEvictionAcWatermark, + double highEvictionAcWatermark, + uint64_t maxEvictionBatch, + uint64_t minEvictionBatch); + ~FreeThresholdStrategy() {} + + std::vector calculateBatchSizes( + const CacheBase& cache, + std::vector acVecs); + + private: + double lowEvictionAcWatermark{2.0}; + double highEvictionAcWatermark{5.0}; + uint64_t maxEvictionBatch{40}; + uint64_t minEvictionBatch{5}; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/MMLru-inl.h b/cachelib/allocator/MMLru-inl.h index 842d87ddb8..751bcca5c1 100644 --- a/cachelib/allocator/MMLru-inl.h +++ b/cachelib/allocator/MMLru-inl.h @@ -229,17 +229,17 @@ void MMLru::Container::withEvictionIterator(F&& fun) { } } -//template T::*HookPtr> -//template -//void -//MMLru::Container::withPromotionIterator(F&& fun) { -// if (config_.useCombinedLockForIterators) { -// lruMutex_->lock_combine([this, &fun]() { fun(Iterator{lru_.begin()}); }); -// } else { -// LockHolder lck{*lruMutex_}; -// fun(Iterator{lru_.begin()}); -// } -//} +template T::*HookPtr> +template +void +MMLru::Container::withPromotionIterator(F&& fun) { + if (config_.useCombinedLockForIterators) { + lruMutex_->lock_combine([this, &fun]() { fun(Iterator{lru_.begin()}); }); + } else { + LockHolder lck{*lruMutex_}; + fun(Iterator{lru_.begin()}); + } +} template T::*HookPtr> void MMLru::Container::ensureNotInsertionPoint(T& node) noexcept { diff --git a/cachelib/allocator/MMLru.h b/cachelib/allocator/MMLru.h index 645b8f0e86..cf3253349a 100644 --- a/cachelib/allocator/MMLru.h +++ b/cachelib/allocator/MMLru.h @@ -377,6 +377,9 @@ class MMLru { template void withEvictionIterator(F&& f); + template + void withPromotionIterator(F&& f); + // get copy of current config Config getConfig() const; diff --git a/cachelib/allocator/MMTinyLFU-inl.h b/cachelib/allocator/MMTinyLFU-inl.h index 46640b24ca..9203a54dd6 100644 --- a/cachelib/allocator/MMTinyLFU-inl.h +++ b/cachelib/allocator/MMTinyLFU-inl.h @@ -227,6 +227,13 @@ void MMTinyLFU::Container::withEvictionIterator(F&& fun) { fun(getEvictionIterator()); } +template T::*HookPtr> +template +void +MMTinyLFU::Container::withPromotionIterator(F&& fun) { + throw std::runtime_error("Not supported"); +} + template T::*HookPtr> void MMTinyLFU::Container::removeLocked(T& node) noexcept { if (isTiny(node)) { diff --git a/cachelib/allocator/MMTinyLFU.h b/cachelib/allocator/MMTinyLFU.h index c8f2699264..eb45cefd22 100644 --- a/cachelib/allocator/MMTinyLFU.h +++ b/cachelib/allocator/MMTinyLFU.h @@ -496,6 +496,9 @@ class MMTinyLFU { // iterator passed as parameter. template void withEvictionIterator(F&& f); + + template + void withPromotionIterator(F&& f); // for saving the state of the lru // diff --git a/cachelib/allocator/PromotionStrategy.h b/cachelib/allocator/PromotionStrategy.h new file mode 100644 index 0000000000..1022aca0f8 --- /dev/null +++ b/cachelib/allocator/PromotionStrategy.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cachelib/allocator/BackgroundMoverStrategy.h" +#include "cachelib/allocator/Cache.h" + +namespace facebook { +namespace cachelib { + +// Base class for background eviction strategy. +class PromotionStrategy : public BackgroundMoverStrategy { + public: + PromotionStrategy(uint64_t promotionAcWatermark, + uint64_t maxPromotionBatch, + uint64_t minPromotionBatch) + : promotionAcWatermark(promotionAcWatermark), + maxPromotionBatch(maxPromotionBatch), + minPromotionBatch(minPromotionBatch) {} + ~PromotionStrategy() {} + + std::vector calculateBatchSizes( + const CacheBase& cache, + std::vector acVec) { + std::vector batches{}; + for (auto [tid, pid, cid] : acVec) { + XDCHECK(tid > 0); + auto stats = cache.getACStats(tid - 1, pid, cid); + if ((1-stats.usageFraction())*100 < promotionAcWatermark) + batches.push_back(0); + else { + auto maxPossibleItemsToPromote = static_cast( + (promotionAcWatermark - (1-stats.usageFraction())*100) * + (stats.totalSlabs() * Slab::kSize) / stats.allocSize); + batches.push_back(maxPossibleItemsToPromote); + } + } + + if (batches.size() == 0) { + return batches; + } + + auto maxBatch = *std::max_element(batches.begin(), batches.end()); + if (maxBatch == 0) + return batches; + + std::transform( + batches.begin(), batches.end(), batches.begin(), [&](auto numItems) { + if (numItems == 0) { + return 0UL; + } + + auto cappedBatchSize = maxPromotionBatch * numItems / maxBatch; + if (cappedBatchSize < minPromotionBatch) + return minPromotionBatch; + else + return cappedBatchSize; + }); + + return batches; + } + + private: + double promotionAcWatermark{4.0}; + uint64_t maxPromotionBatch{40}; + uint64_t minPromotionBatch{5}; +}; + +} // namespace cachelib +} // namespace facebook diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp index 3b7d9bdf37..9fcc9e496f 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp @@ -26,6 +26,7 @@ using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest; TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersBackgroundMovers ) { this->testMultiTiersBackgroundMovers(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersRemoveDuringEviction) { this->testMultiTiersRemoveDuringEviction(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersReplaceDuringEviction) { this->testMultiTiersReplaceDuringEviction(); } diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h index c336b59a4b..ba604c4886 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h @@ -19,6 +19,8 @@ #include "cachelib/allocator/CacheAllocatorConfig.h" #include "cachelib/allocator/MemoryTierCacheConfig.h" #include "cachelib/allocator/tests/TestBase.h" +#include "cachelib/allocator/FreeThresholdStrategy.h" +#include "cachelib/allocator/PromotionStrategy.h" #include @@ -85,6 +87,71 @@ class AllocatorMemoryTiersTest : public AllocatorTest { ASSERT(handle != nullptr); ASSERT_NO_THROW(alloc->insertOrReplace(handle)); } + + void testMultiTiersBackgroundMovers() { + typename AllocatorT::Config config; + config.setCacheSize(10 * Slab::kSize); + config.enableCachePersistence("/tmp"); + config.usePosixForShm(); + config.configureMemoryTiers({ + MemoryTierCacheConfig::fromShm() + .setRatio(1).setMemBind(std::string("0")), + MemoryTierCacheConfig::fromShm() + .setRatio(1).setMemBind(std::string("0")) + }); + config.enableBackgroundEvictor(std::make_shared(2, 10, 100, 40), + std::chrono::milliseconds(10),1); + config.enableBackgroundPromoter(std::make_shared(5, 4, 2), + std::chrono::milliseconds(10),1); + + auto allocator = std::make_unique(AllocatorT::SharedMemNew, config); + ASSERT(allocator != nullptr); + const size_t numBytes = allocator->getCacheMemoryStats().ramCacheSize; + + auto poolId = allocator->addPool("default", numBytes); + + const unsigned int keyLen = 100; + const unsigned int size = 100; + unsigned int allocs = 0; + + //we should work on pool stats because filluppooluntil evictions + //will finish once we evict an item from tier 0 to tier 1 and + //there will be unallocated memory left. + while (allocs < 174760) { + const auto key = this->getRandomNewKey(*allocator, keyLen); + ASSERT_EQ(allocator->find(key), nullptr); + auto handle = util::allocateAccessible(*allocator, poolId, key, size); + allocs++; + } + + const auto key = this->getRandomNewKey(*allocator, keyLen); + auto handle = util::allocateAccessible(*allocator, poolId, key, size); + ASSERT_NE(nullptr, handle); + const uint8_t cid = allocator->getAllocInfo(handle->getMemory()).classId; + ASSERT_EQ(cid,5); + auto stats = allocator->getGlobalCacheStats(); + auto slabStats = allocator->getACStats(0,0,cid); + const auto& mpStats = allocator->getPoolByTid(poolId, 0).getStats(); + //cache is 10MB should move about 1MB to reach 10% free + uint32_t approxEvict = (1024*1024)/mpStats.acStats.at(cid).allocSize; + while (stats.evictionStats.numMovedItems < approxEvict*0.95 && (1-slabStats.usageFraction()) >= 0.095) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + stats = allocator->getGlobalCacheStats(); + slabStats = allocator->getACStats(0,0,cid); + } + ASSERT_GE(1-slabStats.usageFraction(),0.095); + + auto perclassEstats = allocator->getBackgroundMoverClassStats(MoverDir::Evict); + auto perclassPstats = allocator->getBackgroundMoverClassStats(MoverDir::Promote); + + ASSERT_GE(stats.evictionStats.numMovedItems,1); + ASSERT_GE(stats.evictionStats.runCount,1); + ASSERT_GE(stats.promotionStats.numMovedItems,1); + + ASSERT_GE(perclassEstats[0][0][cid], 1); + ASSERT_GE(perclassPstats[1][0][cid], 1); + + } void testMultiTiersValidMixed() { typename AllocatorT::Config config; diff --git a/cachelib/allocator/tests/CacheBaseTest.cpp b/cachelib/allocator/tests/CacheBaseTest.cpp index e7778d6ccf..dae14c5335 100644 --- a/cachelib/allocator/tests/CacheBaseTest.cpp +++ b/cachelib/allocator/tests/CacheBaseTest.cpp @@ -33,6 +33,8 @@ class CacheBaseTest : public CacheBase, public SlabAllocatorTestBase { const std::string getCacheName() const override { return cacheName; } bool isObjectCache() const override { return false; } const MemoryPool& getPool(PoolId) const override { return memoryPool_; } + //TODO: support tiers + const MemoryPool& getPoolByTid(PoolId, TierId tid) const override { return memoryPool_; } PoolStats getPoolStats(PoolId) const override { return PoolStats(); } ACStats getACStats(TierId, PoolId, ClassId) const { return ACStats(); }; AllSlabReleaseEvents getAllSlabReleaseEvents(PoolId) const override { diff --git a/cachelib/cachebench/cache/Cache-inl.h b/cachelib/cachebench/cache/Cache-inl.h index 08c9d05e39..3f54fd31e4 100644 --- a/cachelib/cachebench/cache/Cache-inl.h +++ b/cachelib/cachebench/cache/Cache-inl.h @@ -46,6 +46,16 @@ Cache::Cache(const CacheConfig& config, config_.getRebalanceStrategy(), std::chrono::seconds(config_.poolRebalanceIntervalSec)); + allocatorConfig_.enableBackgroundEvictor( + config_.getBackgroundEvictorStrategy(), + std::chrono::milliseconds(config_.backgroundEvictorIntervalMilSec), + config_.evictorThreads); + + allocatorConfig_.enableBackgroundPromoter( + config_.getBackgroundPromoterStrategy(), + std::chrono::milliseconds(config_.backgroundPromoterIntervalMilSec), + config_.promoterThreads); + if (config_.moveOnSlabRelease && movingSync != nullptr) { allocatorConfig_.enableMovingOnSlabRelease( [](Item& oldItem, Item& newItem, Item* parentPtr) { @@ -100,6 +110,12 @@ Cache::Cache(const CacheConfig& config, } }); + allocatorConfig_.maxEvictionBatch = config_.maxEvictionBatch; + allocatorConfig_.maxPromotionBatch = config_.maxPromotionBatch; + allocatorConfig_.minEvictionBatch = config_.minEvictionBatch; + allocatorConfig_.minPromotionBatch = config_.minPromotionBatch; + allocatorConfig_.maxEvictionPromotionHotness = config_.maxEvictionPromotionHotness; + if (config_.enableItemDestructorCheck) { auto removeCB = [&](const typename Allocator::DestructorData& data) { if (!itemRecords_.validate(data)) { @@ -660,6 +676,21 @@ Stats Cache::getStats() const { const auto navyStats = cache_->getNvmCacheStatsMap().toMap(); ret.allocationClassStats = allocationClassStats; + + ret.backgndEvicStats.nEvictedItems = + cacheStats.evictionStats.numMovedItems; + ret.backgndEvicStats.nTraversals = + cacheStats.evictionStats.runCount; + ret.backgndEvicStats.nClasses = + cacheStats.evictionStats.totalClasses; + ret.backgndEvicStats.evictionSize = + cacheStats.evictionStats.totalBytesMoved; + + ret.backgndPromoStats.nPromotedItems = + cacheStats.promotionStats.numMovedItems; + ret.backgndPromoStats.nTraversals = + cacheStats.promotionStats.runCount; + ret.numEvictions = aggregate.numEvictions(); ret.numItems = aggregate.numItems(); ret.evictAttempts = cacheStats.evictionAttempts; @@ -713,6 +744,9 @@ Stats Cache::getStats() const { ret.nvmCounters = cache_->getNvmCacheStatsMap().toMap(); } + ret.backgroundEvictionClasses = cache_->getBackgroundMoverClassStats(MoverDir::Evict); + ret.backgroundPromotionClasses = cache_->getBackgroundMoverClassStats(MoverDir::Promote); + // nvm stats from navy if (!isRamOnly() && !navyStats.empty()) { auto lookup = [&navyStats](const std::string& key) { diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index 6d118f0d24..706dfdf67a 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -26,7 +26,33 @@ DECLARE_string(report_ac_memory_usage_stats); namespace facebook { namespace cachelib { namespace cachebench { + +struct BackgroundEvictionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t nEvictedItems{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t nTraversals{0}; + + // number of classes + uint64_t nClasses{0}; + + // size of evicted items + uint64_t evictionSize{0}; +}; + +struct BackgroundPromotionStats { + // the number of items this worker evicted by looking at pools/classes stats + uint64_t nPromotedItems{0}; + + // number of times we went executed the thread //TODO: is this def correct? + uint64_t nTraversals{0}; +}; + struct Stats { + BackgroundEvictionStats backgndEvicStats; + BackgroundPromotionStats backgndPromoStats; + uint64_t numEvictions{0}; uint64_t numItems{0}; @@ -108,6 +134,9 @@ struct Stats { // cachebench. std::unordered_map nvmCounters; + std::map>> backgroundEvictionClasses; + std::map>> backgroundPromotionClasses; + // errors from the nvm engine. std::unordered_map nvmErrors; @@ -127,6 +156,16 @@ struct Stats { << std::endl; out << folly::sformat("RAM Evictions : {:,}", numEvictions) << std::endl; + auto foreachAC = [&](auto &map, auto cb) { + for (auto &tidStats : map) { + for (auto &pidStat : tidStats.second) { + for (auto &cidStat : pidStat.second) { + cb(tidStats.first, pidStat.first, cidStat.first, cidStat.second); + } + } + } + }; + for (auto pid = 0U; pid < poolUsageFraction.size(); pid++) { out << folly::sformat("Fraction of pool {:,} used : {:.2f}", pid, poolUsageFraction[pid]) @@ -187,6 +226,10 @@ struct Stats { }); } + out << folly::sformat("Tier 0 Background Evicted Items : {:,}", + backgndEvicStats.nEvictedItems) << std::endl; + out << folly::sformat("Tier 0 Background Traversals : {:,}", + backgndEvicStats.nTraversals) << std::endl; if (numCacheGets > 0) { out << folly::sformat("Cache Gets : {:,}", numCacheGets) << std::endl; out << folly::sformat("Hit Ratio : {:6.2f}%", overallHitRatio) @@ -217,6 +260,22 @@ struct Stats { } } + if (!backgroundEvictionClasses.empty() && backgndEvicStats.nEvictedItems > 0 ) { + out << "== Class Background Eviction Counters Map ==" << std::endl; + foreachAC(backgroundEvictionClasses, [&](auto tid, auto pid, auto cid, auto evicted){ + out << folly::sformat("tid{:2} pid{:2} cid{:4} evicted: {:4}", + tid, pid, cid, evicted) << std::endl; + }); + } + + if (!backgroundPromotionClasses.empty() && backgndPromoStats.nPromotedItems > 0) { + out << "== Class Background Promotion Counters Map ==" << std::endl; + foreachAC(backgroundPromotionClasses, [&](auto tid, auto pid, auto cid, auto promoted){ + out << folly::sformat("tid{:2} pid{:2} cid{:4} promoted: {:4}", + tid, pid, cid, promoted) << std::endl; + }); + } + if (numNvmGets > 0 || numNvmDeletes > 0 || numNvmPuts > 0) { const double ramHitRatio = invertPctFn(numCacheGetMiss, numCacheGets); const double nvmHitRatio = invertPctFn(numNvmGetMiss, numNvmGets); diff --git a/cachelib/cachebench/util/CacheConfig.cpp b/cachelib/cachebench/util/CacheConfig.cpp index 0676a4ab67..5072021f9e 100644 --- a/cachelib/cachebench/util/CacheConfig.cpp +++ b/cachelib/cachebench/util/CacheConfig.cpp @@ -19,6 +19,8 @@ #include "cachelib/allocator/HitsPerSlabStrategy.h" #include "cachelib/allocator/LruTailAgeStrategy.h" #include "cachelib/allocator/RandomStrategy.h" +#include "cachelib/allocator/FreeThresholdStrategy.h" +#include "cachelib/allocator/PromotionStrategy.h" namespace facebook { namespace cachelib { @@ -28,6 +30,9 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, cacheDir); JSONSetVal(configJson, cacheSizeMB); JSONSetVal(configJson, poolRebalanceIntervalSec); + JSONSetVal(configJson, backgroundEvictorIntervalMilSec); + JSONSetVal(configJson, backgroundPromoterIntervalMilSec); + JSONSetVal(configJson, backgroundEvictorStrategy); JSONSetVal(configJson, moveOnSlabRelease); JSONSetVal(configJson, rebalanceStrategy); JSONSetVal(configJson, rebalanceMinSlabs); @@ -102,10 +107,27 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, nvmAdmissionRetentionTimeThreshold); JSONSetVal(configJson, customConfigJson); + + //Background related configs + JSONSetVal(configJson, lowEvictionAcWatermark); + JSONSetVal(configJson, highEvictionAcWatermark); + JSONSetVal(configJson, minAcAllocationWatermark); + JSONSetVal(configJson, maxAcAllocationWatermark); + JSONSetVal(configJson, numDuplicateElements); + JSONSetVal(configJson, syncPromotion); + JSONSetVal(configJson, evictorThreads); + JSONSetVal(configJson, promoterThreads); + JSONSetVal(configJson, promotionAcWatermark); + JSONSetVal(configJson, maxEvictionBatch); + JSONSetVal(configJson, maxPromotionBatch); + JSONSetVal(configJson, minEvictionBatch); + JSONSetVal(configJson, minPromotionBatch); + JSONSetVal(configJson, maxEvictionPromotionHotness); + // if you added new fields to the configuration, update the JSONSetVal // to make them available for the json configs and increment the size // below - checkCorrectSize(); + checkCorrectSize(); if (numPools != poolSizes.size()) { throw std::invalid_argument(folly::sformat( @@ -141,6 +163,20 @@ MemoryTierConfig::MemoryTierConfig(const folly::dynamic& configJson) { checkCorrectSize(); } + +std::shared_ptr CacheConfig::getBackgroundEvictorStrategy() const { + if (backgroundEvictorIntervalMilSec == 0) { + return nullptr; + } + return std::make_shared(lowEvictionAcWatermark, highEvictionAcWatermark, maxEvictionBatch, minEvictionBatch); +} + +std::shared_ptr CacheConfig::getBackgroundPromoterStrategy() const { + if (backgroundPromoterIntervalMilSec == 0) { + return nullptr; + } + return std::make_shared(promotionAcWatermark, maxPromotionBatch, minPromotionBatch); +} } // namespace cachebench } // namespace cachelib } // namespace facebook diff --git a/cachelib/cachebench/util/CacheConfig.h b/cachelib/cachebench/util/CacheConfig.h index 0240fc4160..411fe73275 100644 --- a/cachelib/cachebench/util/CacheConfig.h +++ b/cachelib/cachebench/util/CacheConfig.h @@ -20,6 +20,7 @@ #include "cachelib/allocator/CacheAllocator.h" #include "cachelib/allocator/RebalanceStrategy.h" +#include "cachelib/allocator/BackgroundMoverStrategy.h" #include "cachelib/cachebench/util/JSONConfig.h" #include "cachelib/common/Ticker.h" #include "cachelib/navy/common/Device.h" @@ -71,7 +72,10 @@ struct CacheConfig : public JSONConfig { uint64_t cacheSizeMB{0}; uint64_t poolRebalanceIntervalSec{0}; + uint64_t backgroundEvictorIntervalMilSec{0}; + uint64_t backgroundPromoterIntervalMilSec{0}; std::string rebalanceStrategy; + std::string backgroundEvictorStrategy; uint64_t rebalanceMinSlabs{1}; double rebalanceDiffRatio{0.25}; bool moveOnSlabRelease{false}; @@ -252,6 +256,27 @@ struct CacheConfig : public JSONConfig { // eviction-age is more than this threshold. 0 means no threshold uint32_t nvmAdmissionRetentionTimeThreshold{0}; + // See BackgroundMovers.md for complete description + double promotionAcWatermark{4.0}; + double lowEvictionAcWatermark{2.0}; + double highEvictionAcWatermark{5.0}; + double minAcAllocationWatermark{0.0}; + double maxAcAllocationWatermark{0.0}; + + double numDuplicateElements{0.0}; // inclusivness of the cache + double syncPromotion{0.0}; // can promotion be done synchronously in user thread + + uint64_t evictorThreads{1}; + uint64_t promoterThreads{1}; + + uint64_t maxEvictionBatch{40}; + uint64_t maxPromotionBatch{10}; + + uint64_t minEvictionBatch{5}; + uint64_t minPromotionBatch{5}; + + uint64_t maxEvictionPromotionHotness{60}; + // // Options below are not to be populated with JSON // @@ -287,6 +312,8 @@ struct CacheConfig : public JSONConfig { CacheConfig() {} std::shared_ptr getRebalanceStrategy() const; + std::shared_ptr getBackgroundEvictorStrategy() const; + std::shared_ptr getBackgroundPromoterStrategy() const; }; } // namespace cachebench } // namespace cachelib From 6203a959632620fd42c82147cebe220e21e92f67 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Thu, 16 Feb 2023 14:19:21 -0800 Subject: [PATCH 16/31] fix race in moveRegularItemWith sync where insertOrReplace can cause move to fail - updated slab release logic for move failure, but there is still an issue with slab movement. currently investigating. --- cachelib/allocator/CacheAllocator-inl.h | 98 +++++++++++--- cachelib/allocator/CacheAllocator.h | 2 +- cachelib/allocator/CacheItem.h | 5 + .../tests/AllocatorMemoryTiersTest.cpp | 3 +- .../tests/AllocatorMemoryTiersTest.h | 128 +++++++++++++++++- 5 files changed, 212 insertions(+), 24 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 47ba92eb94..3be01469de 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -1293,8 +1293,21 @@ size_t CacheAllocator::wakeUpWaitersLocked(folly::StringPiece key, } template -void CacheAllocator::moveRegularItemWithSync( +bool CacheAllocator::moveRegularItemWithSync( Item& oldItem, WriteHandle& newItemHdl) { + //on function exit - the new item handle is no longer moving + //and other threads may access it - but in case where + //we failed to replace in access container we can give the + //new item back to the allocator + auto guard = folly::makeGuard([&]() { + auto ref = newItemHdl->unmarkMoving(); + if (UNLIKELY(ref == 0)) { + const auto res = + releaseBackToAllocator(*newItemHdl, RemoveContext::kNormal, false); + XDCHECK(res == ReleaseRes::kReleased); + } + }); + XDCHECK(oldItem.isMoving()); XDCHECK(!oldItem.isExpired()); // TODO: should we introduce new latency tracker. E.g. evictRegularLatency_ @@ -1325,6 +1338,22 @@ void CacheAllocator::moveRegularItemWithSync( auto replaced = accessContainer_->replaceIf(oldItem, *newItemHdl, predicate); + // another thread may have called insertOrReplace which could have + // marked this item as unaccessible causing the replaceIf + // in the access container to fail - in this case we want + // to abort the move since the item is no longer valid + if (!replaced) { + return false; + } + // what if another thread calls insertOrReplace now when + // the item is moving and already replaced in the hash table? + // 1. it succeeds in updating the hash table - so there is + // no guarentee that isAccessible() is true + // 2. it will then try to remove from MM container + // - this operation will wait for newItemHdl to + // be unmarkedMoving via the waitContext + // 3. replaced handle is returned and eventually drops + // ref to 0 and the item is recycled back to allocator. if (config_.moveCb) { // Execute the move callback. We cannot make any guarantees about the @@ -1366,14 +1395,7 @@ void CacheAllocator::moveRegularItemWithSync( XDCHECK(newItemHdl->hasChainedItem()); } newItemHdl.unmarkNascent(); - auto ref = newItemHdl->unmarkMoving(); - //remove because there is a chance the new item was not - //added to the access container - if (UNLIKELY(ref == 0)) { - const auto res = - releaseBackToAllocator(*newItemHdl, RemoveContext::kNormal, false); - XDCHECK(res == ReleaseRes::kReleased); - } + return true; } template @@ -1626,28 +1648,43 @@ CacheAllocator::getNextCandidate(TierId tid, auto evictedToNext = lastTier ? nullptr : tryEvictToNextMemoryTier(*candidate, false); if (!evictedToNext) { - if (!token.isValid()) { + //if insertOrReplace was called during move + //then candidate will not be accessible (failed replace during tryEvict) + // - therefore this was why we failed to + // evict to the next tier and insertOrReplace + // will remove from NVM cache + //however, if candidate is accessible + //that means the allocation in the next + //tier failed - so we will continue to + //evict the item to NVM cache + bool failedToReplace = !candidate->isAccessible(); + if (!token.isValid() && !failedToReplace) { token = createPutToken(*candidate); } - // tryEvictToNextMemoryTier should only fail if allocation of the new item fails - // in that case, it should be still possible to mark item as exclusive. + // tryEvictToNextMemoryTier can fail if: + // a) allocation of the new item fails in that case, + // it should be still possible to mark item for eviction. + // b) another thread calls insertOrReplace and the item + // is no longer accessible // // in case that we are on the last tier, we whould have already marked // as exclusive since we will not be moving the item to the next tier // but rather just evicting all together, no need to - // markExclusiveWhenMoving + // markForEvictionWhenMoving auto ret = lastTier ? true : candidate->markForEvictionWhenMoving(); XDCHECK(ret); unlinkItemForEviction(*candidate); + + if (token.isValid() && shouldWriteToNvmCacheExclusive(*candidate) + && !failedToReplace) { + nvmCache_->put(*candidate, std::move(token)); + } // wake up any readers that wait for the move to complete // it's safe to do now, as we have the item marked exclusive and // no other reader can be added to the waiters list wakeUpWaiters(*candidate, {}); - if (token.isValid() && shouldWriteToNvmCacheExclusive(*candidate)) { - nvmCache_->put(*candidate, std::move(token)); - } } else { XDCHECK(!evictedToNext->isMarkedForEviction() && !evictedToNext->isMoving()); XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); @@ -1776,7 +1813,10 @@ CacheAllocator::tryEvictToNextMemoryTier( if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); - moveRegularItemWithSync(item, newItemHdl); + if (!moveRegularItemWithSync(item, newItemHdl)) { + return WriteHandle{}; + } + XDCHECK_EQ(newItemHdl->getKey(),item.getKey()); item.unmarkMoving(); return newItemHdl; } else { @@ -1815,7 +1855,9 @@ CacheAllocator::tryPromoteToNextMemoryTier( if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); - moveRegularItemWithSync(item, newItemHdl); + if (!moveRegularItemWithSync(item, newItemHdl)) { + return WriteHandle{}; + } item.unmarkMoving(); return newItemHdl; } else { @@ -3175,9 +3217,23 @@ bool CacheAllocator::tryMovingForSlabRelease( // TODO: add support for chained items return false; } else { - moveRegularItemWithSync(oldItem, newItemHdl); - removeFromMMContainer(oldItem); - return true; + //move can fail if another thread calls insertOrReplace + //in this case oldItem is no longer valid (not accessible, + //it gets removed from MMContainer and evictForSlabRelease + //will send it back to the allocator + bool ret = moveRegularItemWithSync(oldItem, newItemHdl); + if (!ret) { + //we failed to move - newItemHdl was released back to allocator + //by the moveRegularItemWithSync but oldItem is not accessible + //and no longer valid - we need to clean it up here + XDCHECK(!oldItem.isAccessible()); + oldItem.markForEvictionWhenMoving(); + unlinkItemForEviction(oldItem); + wakeUpWaiters(oldItem, {}); + } else { + removeFromMMContainer(oldItem); + } + return ret; } } } diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 597e81f577..e2dafcae3b 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1623,7 +1623,7 @@ class CacheAllocator : public CacheBase { // // @return true If the move was completed, and the containers were updated // successfully. - void moveRegularItemWithSync(Item& oldItem, WriteHandle& newItemHdl); + bool moveRegularItemWithSync(Item& oldItem, WriteHandle& newItemHdl); // Moves a regular item to a different slab. This should only be used during // slab release after the item's exclusive bit has been set. The user supplied diff --git a/cachelib/allocator/CacheItem.h b/cachelib/allocator/CacheItem.h index b4fa339b57..6728b654eb 100644 --- a/cachelib/allocator/CacheItem.h +++ b/cachelib/allocator/CacheItem.h @@ -46,6 +46,9 @@ class BaseAllocatorTest; template class AllocatorHitStatsTest; +template +class AllocatorMemoryTiersTest; + template class MapTest; @@ -473,6 +476,8 @@ class CACHELIB_PACKED_ATTR CacheItem { FRIEND_TEST(ItemTest, NonStringKey); template friend class facebook::cachelib::tests::AllocatorHitStatsTest; + template + friend class facebook::cachelib::tests::AllocatorMemoryTiersTest; }; // A chained item has a hook pointing to the next chained item. The hook is diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp index 9fcc9e496f..1db7fce112 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp @@ -21,7 +21,7 @@ namespace cachelib { namespace tests { using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest; - +//using LruTestAllocatorMemoryTiersTest = AllocatorMemoryTiersTest; // TODO(MEMORY_TIER): add more tests with different eviction policies TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); } @@ -29,6 +29,7 @@ TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiers TEST_F(LruAllocatorMemoryTiersTest, MultiTiersBackgroundMovers ) { this->testMultiTiersBackgroundMovers(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersRemoveDuringEviction) { this->testMultiTiersRemoveDuringEviction(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersReplaceDuringEviction) { this->testMultiTiersReplaceDuringEviction(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersReplaceDuringEvictionWithReader) { this->testMultiTiersReplaceDuringEvictionWithReader(); } } // end of namespace tests } // end of namespace cachelib diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h index ba604c4886..616de6e9db 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h @@ -22,6 +22,10 @@ #include "cachelib/allocator/FreeThresholdStrategy.h" #include "cachelib/allocator/PromotionStrategy.h" +#include +#include +#include +#include #include namespace facebook { @@ -58,6 +62,7 @@ class AllocatorMemoryTiersTest : public AllocatorTest { ASSERT_NO_THROW(alloc->insertOrReplace(handle)); } } + public: void testMultiTiersInvalid() { typename AllocatorT::Config config; @@ -201,7 +206,7 @@ class AllocatorMemoryTiersTest : public AllocatorTest { t->join(); } - + void testMultiTiersReplaceDuringEviction() { std::unique_ptr alloc; PoolId pool; @@ -234,6 +239,127 @@ class AllocatorMemoryTiersTest : public AllocatorTest { testMultiTiersAsyncOpDuringMove(alloc, pool, quit, moveCb); t->join(); + + } + + + void gdb_sync1() {} + void gdb_sync2() {} + void gdb_sync3() {} + using ReadHandle = typename AllocatorT::ReadHandle; + void testMultiTiersReplaceDuringEvictionWithReader() { + sem_unlink ("/gdb1_sem"); + sem_t *sem = sem_open ("/gdb1_sem", O_CREAT | O_EXCL, S_IRUSR | S_IWUSR, 0); + int gdbfd = open("/tmp/gdb1.gdb",O_CREAT | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR); + char gdbcmds[] = + "set attached=1\n" + "break gdb_sync1\n" + "break gdb_sync2\n" + "break moveRegularItemWithSync\n" + "c\n" + "set scheduler-locking on\n" + "thread 1\n" + "c\n" + "thread 4\n" + "c\n" + "thread 5\n" + "break nativeFutexWaitImpl thread 5\n" + "c\n" + "thread 4\n" + "break nativeFutexWaitImpl thread 4\n" + "c\n" + "thread 1\n" + "break releaseBackToAllocator\n" + "c\n" + "c\n" + "thread 5\n" + "c\n" + "thread 4\n" + "c\n" + "thread 1\n" + "break gdb_sync3\n" + "c\n" + "quit\n"; + int ret = write(gdbfd,gdbcmds,strlen(gdbcmds)); + int ppid = getpid(); //parent pid + //int pid = 0; + int pid = fork(); + if (pid == 0) { + sem_wait(sem); + sem_close(sem); + sem_unlink("/gdb1_sem"); + char cmdpid[256]; + sprintf(cmdpid,"%d",ppid); + int f = execlp("gdb","gdb","--pid",cmdpid,"--batch-silent","--command=/tmp/gdb1.gdb",(char*) 0); + ASSERT(f != -1); + } + sem_post(sem); + //wait for gdb to run + int attached = 0; + while (attached == 0); + + std::unique_ptr alloc; + PoolId pool; + bool quit = false; + + typename AllocatorT::Config config; + config.setCacheSize(4 * Slab::kSize); + config.enableCachePersistence("/tmp"); + config.configureMemoryTiers({ + MemoryTierCacheConfig::fromShm() + .setRatio(1).setMemBind(std::string("0")), + MemoryTierCacheConfig::fromShm() + .setRatio(1).setMemBind(std::string("0")) + }); + + alloc = std::make_unique(AllocatorT::SharedMemNew, config); + ASSERT(alloc != nullptr); + pool = alloc->addPool("default", alloc->getCacheMemoryStats().ramCacheSize); + + int i = 0; + typename AllocatorT::Item* evicted; + std::unique_ptr t; + std::unique_ptr r; + while(!quit) { + auto handle = alloc->allocate(pool, std::to_string(++i), std::string("value").size()); + ASSERT(handle != nullptr); + if (i == 1) { + evicted = static_cast(handle.get()); + folly::Latch latch_t(1); + t = std::make_unique([&](){ + auto handleNew = alloc->allocate(pool, std::to_string(1), std::string("new value").size()); + ASSERT(handleNew != nullptr); + latch_t.count_down(); + //first breakpoint will be this one because + //thread 1 still has more items to fill up the + //cache before an evict is evicted + gdb_sync1(); + ASSERT(evicted->isMoving()); + //need to suspend thread 1 - who is doing the eviction + //gdb will do this for us + folly::Latch latch(1); + r = std::make_unique([&](){ + ASSERT(evicted->isMoving()); + latch.count_down(); + auto handleEvict = alloc->find(std::to_string(1)); + //does find block until done moving?? yes + while (evicted->isMarkedForEviction()); //move will fail + XDCHECK(handleEvict == nullptr) << handleEvict->toString(); + ASSERT(handleEvict == nullptr); + }); + latch.wait(); + gdb_sync2(); + alloc->insertOrReplace(handleNew); + ASSERT(!evicted->isAccessible()); //move failed + quit = true; + }); + latch_t.wait(); + } + ASSERT_NO_THROW(alloc->insertOrReplace(handle)); + } + t->join(); + r->join(); + gdb_sync3(); } }; } // namespace tests From 6abb49813da1b8a8724a11901130ab7b560ec8be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Chor=C4=85=C5=BCewicz?= Date: Thu, 16 Mar 2023 05:07:00 -0700 Subject: [PATCH 17/31] Fix race in acquire (#68) The assumption for moving items was that once item is unmarked no one can add new waiters for that item. However, since incrementing item ref count was not done under the MoveMap lock, there was a race: item could have been unmarked right after incRef returned incFailedMoving. --- cachelib/allocator/CacheAllocator-inl.h | 36 ++++++++++++++++--------- cachelib/allocator/CacheAllocator.h | 2 +- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 3be01469de..9071e41b64 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -1023,16 +1023,21 @@ CacheAllocator::acquire(Item* it) { SCOPE_FAIL { stats_.numRefcountOverflow.inc(); }; // TODO: do not block incRef for child items to avoid deadlock - auto failIfMoving = getNumTiers() > 1 && !it->isChainedItem(); - auto incRes = incRef(*it, failIfMoving); - if (LIKELY(incRes == RefcountWithFlags::incResult::incOk)) { - return WriteHandle{it, *this}; - } else if (incRes == RefcountWithFlags::incResult::incFailedEviction){ - // item is being evicted - return WriteHandle{}; - } else { - // item is being moved - wait for completion - return handleWithWaitContextForMovingItem(*it); + const auto failIfMoving = getNumTiers() > 1 && !it->isChainedItem(); + + while (true) { + auto incRes = incRef(*it, failIfMoving); + if (LIKELY(incRes == RefcountWithFlags::incResult::incOk)) { + return WriteHandle{it, *this}; + } else if (incRes == RefcountWithFlags::incResult::incFailedEviction){ + // item is being evicted + return WriteHandle{}; + } else { + // item is being moved - wait for completion + WriteHandle handle; + if (tryGetHandleWithWaitContextForMovingItem(*it, handle)) + return handle; + } } } @@ -1254,20 +1259,25 @@ CacheAllocator::insertOrReplace(const WriteHandle& handle) { * 3. Wait until the moving thread will complete its job. */ template -typename CacheAllocator::WriteHandle -CacheAllocator::handleWithWaitContextForMovingItem(Item& item) { +bool +CacheAllocator::tryGetHandleWithWaitContextForMovingItem(Item& item, WriteHandle& handle) { auto shard = getShardForKey(item.getKey()); auto& movesMap = getMoveMapForShard(shard); { auto lock = getMoveLockForShard(shard); + // item might have been evicted or moved before the lock was acquired + if (!item.isMoving()) + return false; + WriteHandle hdl{*this}; auto waitContext = hdl.getItemWaitContext(); auto ret = movesMap.try_emplace(item.getKey(), std::make_unique()); ret.first->second->addWaiter(std::move(waitContext)); - return hdl; + handle = std::move(hdl); + return true; } } diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index e2dafcae3b..1de39104ea 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -2306,7 +2306,7 @@ auto& mmContainer = getMMContainer(tid, pid, cid); size_t memoryTierSize(TierId tid) const; - WriteHandle handleWithWaitContextForMovingItem(Item& item); + bool tryGetHandleWithWaitContextForMovingItem(Item& item, WriteHandle& handle); size_t wakeUpWaitersLocked(folly::StringPiece key, WriteHandle&& handle); From add2e5fd4c850ebe116d269c962c7146bce6d27e Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Thu, 23 Mar 2023 14:18:14 -0400 Subject: [PATCH 18/31] Per tier pool stats (#70) --- cachelib/allocator/Cache.cpp | 10 +- cachelib/allocator/CacheAllocator-inl.h | 156 ++++++++++++++---- cachelib/allocator/CacheAllocator.h | 6 +- cachelib/allocator/CacheStats.cpp | 92 +++++++++-- cachelib/allocator/CacheStats.h | 46 ++++-- cachelib/allocator/CacheStatsInternal.h | 25 +-- .../tests/AllocatorMemoryTiersTest.cpp | 1 + .../tests/AllocatorMemoryTiersTest.h | 56 +++++++ cachelib/allocator/tests/TestBase-inl.h | 24 +++ cachelib/allocator/tests/TestBase.h | 5 + cachelib/cachebench/cache/Cache-inl.h | 31 ++-- cachelib/cachebench/cache/CacheStats.h | 92 +++++++---- 12 files changed, 419 insertions(+), 125 deletions(-) diff --git a/cachelib/allocator/Cache.cpp b/cachelib/allocator/Cache.cpp index 058eb84501..b871a04782 100644 --- a/cachelib/allocator/Cache.cpp +++ b/cachelib/allocator/Cache.cpp @@ -245,6 +245,7 @@ void CacheBase::updateGlobalCacheStats(const std::string& statPrefix) const { statPrefix + "cache.size.configured", memStats.configuredRamCacheSize + memStats.nvmCacheSize); + //TODO: add specific per-tier counters const auto stats = getGlobalCacheStats(); // Eviction Stats @@ -254,7 +255,8 @@ void CacheBase::updateGlobalCacheStats(const std::string& statPrefix) const { // from both ram and nvm, this is counted as a single eviction from cache. // Ram Evictions: item evicted from ram but it can be inserted into nvm const std::string ramEvictionKey = statPrefix + "ram.evictions"; - counters_.updateDelta(ramEvictionKey, stats.numEvictions); + counters_.updateDelta(ramEvictionKey, + std::accumulate(stats.numEvictions.begin(), stats.numEvictions.end(), 0)); // Nvm Evictions: item evicted from nvm but it can be still in ram const std::string nvmEvictionKey = statPrefix + "nvm.evictions"; counters_.updateDelta(nvmEvictionKey, stats.numNvmEvictions); @@ -296,11 +298,11 @@ void CacheBase::updateGlobalCacheStats(const std::string& statPrefix) const { } counters_.updateDelta(statPrefix + "cache.alloc_attempts", - stats.allocAttempts); + std::accumulate(stats.allocAttempts.begin(), stats.allocAttempts.end(),0)); counters_.updateDelta(statPrefix + "cache.eviction_attempts", - stats.evictionAttempts); + std::accumulate(stats.evictionAttempts.begin(),stats.evictionAttempts.end(),0)); counters_.updateDelta(statPrefix + "cache.alloc_failures", - stats.allocFailures); + std::accumulate(stats.allocFailures.begin(),stats.allocFailures.end(),0)); counters_.updateDelta(statPrefix + "cache.invalid_allocs", stats.invalidAllocs); diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 9071e41b64..f36fbd57de 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -417,8 +417,7 @@ CacheAllocator::allocateInternalTier(TierId tid, util::RollingLatencyTracker rollTracker{ (*stats_.classAllocLatency)[tid][pid][cid]}; - // TODO: per-tier - (*stats_.allocAttempts)[pid][cid].inc(); + (*stats_.allocAttempts)[tid][pid][cid].inc(); void* memory = allocator_[tid]->allocate(pid, requiredSize); @@ -444,12 +443,12 @@ CacheAllocator::allocateInternalTier(TierId tid, handle = acquire(new (memory) Item(key, size, creationTime, expiryTime)); if (handle) { handle.markNascent(); - (*stats_.fragmentationSize)[pid][cid].add( + (*stats_.fragmentationSize)[tid][pid][cid].add( util::getFragmentation(*this, *handle)); } } else { // failed to allocate memory. - (*stats_.allocFailures)[pid][cid].inc(); // TODO: per-tier + (*stats_.allocFailures)[tid][pid][cid].inc(); // wake up rebalancer if (!config_.poolRebalancerDisableForcedWakeUp && poolRebalancer_) { poolRebalancer_->wakeUp(); @@ -521,16 +520,14 @@ CacheAllocator::allocateChainedItemInternal( util::RollingLatencyTracker rollTracker{ (*stats_.classAllocLatency)[tid][pid][cid]}; - // TODO: per-tier? Right now stats_ are not used in any public periodic - // worker - (*stats_.allocAttempts)[pid][cid].inc(); + (*stats_.allocAttempts)[tid][pid][cid].inc(); void* memory = allocator_[tid]->allocate(pid, requiredSize); if (memory == nullptr) { memory = findEviction(tid, pid, cid); } if (memory == nullptr) { - (*stats_.allocFailures)[pid][cid].inc(); + (*stats_.allocFailures)[tid][pid][cid].inc(); return WriteHandle{}; } @@ -542,7 +539,7 @@ CacheAllocator::allocateChainedItemInternal( if (child) { child.markNascent(); - (*stats_.fragmentationSize)[pid][cid].add( + (*stats_.fragmentationSize)[tid][pid][cid].add( util::getFragmentation(*this, *child)); } @@ -857,7 +854,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, stats_.perPoolEvictionAgeSecs_[allocInfo.poolId].trackValue(refreshTime); } - (*stats_.fragmentationSize)[allocInfo.poolId][allocInfo.classId].sub( + (*stats_.fragmentationSize)[tid][allocInfo.poolId][allocInfo.classId].sub( util::getFragmentation(*this, it)); // Chained items can only end up in this place if the user has allocated @@ -940,7 +937,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, const auto childInfo = allocator_[tid]->getAllocInfo(static_cast(head)); - (*stats_.fragmentationSize)[childInfo.poolId][childInfo.classId].sub( + (*stats_.fragmentationSize)[tid][childInfo.poolId][childInfo.classId].sub( util::getFragmentation(*this, *head)); removeFromMMContainer(*head); @@ -1582,12 +1579,12 @@ CacheAllocator::getNextCandidate(TierId tid, auto& mmContainer = getMMContainer(tid, pid, cid); bool lastTier = tid+1 >= getNumTiers(); - mmContainer.withEvictionIterator([this, pid, cid, &candidate, &toRecycle, + mmContainer.withEvictionIterator([this, tid, pid, cid, &candidate, &toRecycle, &searchTries, &mmContainer, &lastTier, &token](auto&& itr) { if (!itr) { ++searchTries; - (*stats_.evictionAttempts)[pid][cid].inc(); + (*stats_.evictionAttempts)[tid][pid][cid].inc(); return; } @@ -1595,7 +1592,7 @@ CacheAllocator::getNextCandidate(TierId tid, config_.evictionSearchTries > searchTries) && itr) { ++searchTries; - (*stats_.evictionAttempts)[pid][cid].inc(); + (*stats_.evictionAttempts)[tid][pid][cid].inc(); auto* toRecycle_ = itr.get(); auto* candidate_ = @@ -1701,6 +1698,7 @@ CacheAllocator::getNextCandidate(TierId tid, XDCHECK(!candidate->isAccessible()); XDCHECK(candidate->getKey() == evictedToNext->getKey()); + (*stats_.numWritebacks)[tid][pid][cid].inc(); wakeUpWaiters(*candidate, std::move(evictedToNext)); } @@ -1728,9 +1726,9 @@ CacheAllocator::findEviction(TierId tid, PoolId pid, ClassId cid) { // NULL. If `ref` == 0 then it means that we are the last holder of // that item. if (candidate->hasChainedItem()) { - (*stats_.chainedItemEvictions)[pid][cid].inc(); + (*stats_.chainedItemEvictions)[tid][pid][cid].inc(); } else { - (*stats_.regularItemEvictions)[pid][cid].inc(); + (*stats_.regularItemEvictions)[tid][pid][cid].inc(); } if (auto eventTracker = getEventTracker()) { @@ -2333,7 +2331,7 @@ bool CacheAllocator::recordAccessInMMContainer(Item& item, const auto tid = getTierId(item); const auto allocInfo = allocator_[tid]->getAllocInfo(static_cast(&item)); - (*stats_.cacheHits)[allocInfo.poolId][allocInfo.classId].inc(); + (*stats_.cacheHits)[tid][allocInfo.poolId][allocInfo.classId].inc(); // track recently accessed items if needed if (UNLIKELY(config_.trackRecentItemsForDump)) { @@ -2802,6 +2800,8 @@ size_t CacheAllocator::getPoolSize(PoolId poolId) const { template PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { + //this pool ref is just used to get class ids, which will be the + //same across tiers const auto& pool = allocator_[currentTier()]->getPool(poolId); const auto& allocSizes = pool.getAllocSizes(); auto mpStats = pool.getStats(); @@ -2820,24 +2820,42 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { // TODO export evictions, numItems etc from compact cache directly. if (!isCompactCache) { for (const ClassId cid : classIds) { - uint64_t classHits = (*stats_.cacheHits)[poolId][cid].get(); - XDCHECK(mmContainers_[currentTier()][poolId][cid], - folly::sformat("Pid {}, Cid {} not initialized.", poolId, cid)); + uint64_t allocAttempts = 0, evictionAttempts = 0, allocFailures = 0, + fragmentationSize = 0, classHits = 0, chainedItemEvictions = 0, + regularItemEvictions = 0, numWritebacks = 0; + MMContainerStat mmContainerStats; + for (TierId tid = 0; tid < getNumTiers(); tid++) { + allocAttempts += (*stats_.allocAttempts)[tid][poolId][cid].get(); + evictionAttempts += (*stats_.evictionAttempts)[tid][poolId][cid].get(); + allocFailures += (*stats_.allocFailures)[tid][poolId][cid].get(); + fragmentationSize += (*stats_.fragmentationSize)[tid][poolId][cid].get(); + classHits += (*stats_.cacheHits)[tid][poolId][cid].get(); + chainedItemEvictions += (*stats_.chainedItemEvictions)[tid][poolId][cid].get(); + regularItemEvictions += (*stats_.regularItemEvictions)[tid][poolId][cid].get(); + numWritebacks += (*stats_.numWritebacks)[tid][poolId][cid].get(); + mmContainerStats += getMMContainerStat(tid, poolId, cid); + XDCHECK(mmContainers_[tid][poolId][cid], + folly::sformat("Tid {}, Pid {}, Cid {} not initialized.", tid, poolId, cid)); + } cacheStats.insert( {cid, - {allocSizes[cid], (*stats_.allocAttempts)[poolId][cid].get(), - (*stats_.evictionAttempts)[poolId][cid].get(), - (*stats_.allocFailures)[poolId][cid].get(), - (*stats_.fragmentationSize)[poolId][cid].get(), classHits, - (*stats_.chainedItemEvictions)[poolId][cid].get(), - (*stats_.regularItemEvictions)[poolId][cid].get(), - getMMContainerStat(currentTier(), poolId, cid)}}); + {allocSizes[cid], + allocAttempts, + evictionAttempts, + allocFailures, + fragmentationSize, + classHits, + chainedItemEvictions, + regularItemEvictions, + numWritebacks, + mmContainerStats}}); totalHits += classHits; } } PoolStats ret; ret.isCompactCache = isCompactCache; + //pool name is also shared among tiers ret.poolName = allocator_[currentTier()]->getPoolName(poolId); ret.poolSize = pool.getPoolSize(); ret.poolUsableSize = pool.getPoolUsableSize(); @@ -2850,6 +2868,59 @@ PoolStats CacheAllocator::getPoolStats(PoolId poolId) const { return ret; } +template +PoolStats CacheAllocator::getPoolStats(TierId tid, PoolId poolId) const { + const auto& pool = allocator_[tid]->getPool(poolId); + const auto& allocSizes = pool.getAllocSizes(); + auto mpStats = pool.getStats(); + const auto& classIds = mpStats.classIds; + + // check if this is a compact cache. + bool isCompactCache = false; + { + folly::SharedMutex::ReadHolder lock(compactCachePoolsLock_); + isCompactCache = isCompactCachePool_[poolId]; + } + + std::unordered_map cacheStats; + uint64_t totalHits = 0; + // cacheStats is only menaningful for pools that are not compact caches. + // TODO export evictions, numItems etc from compact cache directly. + if (!isCompactCache) { + for (const ClassId cid : classIds) { + uint64_t classHits = (*stats_.cacheHits)[tid][poolId][cid].get(); + XDCHECK(mmContainers_[tid][poolId][cid], + folly::sformat("Tid {}, Pid {}, Cid {} not initialized.", tid, poolId, cid)); + cacheStats.insert( + {cid, + {allocSizes[cid], + (*stats_.allocAttempts)[tid][poolId][cid].get(), + (*stats_.evictionAttempts)[tid][poolId][cid].get(), + (*stats_.allocFailures)[tid][poolId][cid].get(), + (*stats_.fragmentationSize)[tid][poolId][cid].get(), + classHits, + (*stats_.chainedItemEvictions)[tid][poolId][cid].get(), + (*stats_.regularItemEvictions)[tid][poolId][cid].get(), + (*stats_.numWritebacks)[tid][poolId][cid].get(), + getMMContainerStat(tid, poolId, cid)}}); + totalHits += classHits; + } + } + + PoolStats ret; + ret.isCompactCache = isCompactCache; + ret.poolName = allocator_[tid]->getPoolName(poolId); + ret.poolSize = pool.getPoolSize(); + ret.poolUsableSize = pool.getPoolUsableSize(); + ret.poolAdvisedSize = pool.getPoolAdvisedSize(); + ret.cacheStats = std::move(cacheStats); + ret.mpStats = std::move(mpStats); + ret.numPoolGetHits = totalHits; + ret.evictionAgeSecs = stats_.perPoolEvictionAgeSecs_[poolId].estimate(); + + return ret; +} + template ACStats CacheAllocator::getACStats(TierId tid, PoolId poolId, @@ -3100,7 +3171,7 @@ bool CacheAllocator::moveForSlabRelease( const auto allocInfo = allocator_[tid]->getAllocInfo(oldItem.getMemory()); allocator_[tid]->free(&oldItem); - (*stats_.fragmentationSize)[allocInfo.poolId][allocInfo.classId].sub( + (*stats_.fragmentationSize)[tid][allocInfo.poolId][allocInfo.classId].sub( util::getFragmentation(*this, oldItem)); stats_.numMoveSuccesses.inc(); return true; @@ -3379,12 +3450,13 @@ void CacheAllocator::evictForSlabRelease( nvmCache_->put(*evicted, std::move(token)); } + const auto tid = getTierId(*evicted); const auto allocInfo = - allocator_[getTierId(*evicted)]->getAllocInfo(static_cast(evicted)); + allocator_[tid]->getAllocInfo(static_cast(evicted)); if (evicted->hasChainedItem()) { - (*stats_.chainedItemEvictions)[allocInfo.poolId][allocInfo.classId].inc(); + (*stats_.chainedItemEvictions)[tid][allocInfo.poolId][allocInfo.classId].inc(); } else { - (*stats_.regularItemEvictions)[allocInfo.poolId][allocInfo.classId].inc(); + (*stats_.regularItemEvictions)[tid][allocInfo.poolId][allocInfo.classId].inc(); } stats_.numEvictionSuccesses.inc(); @@ -3607,8 +3679,13 @@ folly::IOBufQueue CacheAllocator::saveStateToIOBuf() { for (PoolId pid : pools) { for (unsigned int cid = 0; cid < (*stats_.fragmentationSize)[pid].size(); ++cid) { + uint64_t fragmentationSize = 0; + for (TierId tid = 0; tid < getNumTiers(); tid++) { + fragmentationSize += (*stats_.fragmentationSize)[tid][pid][cid].get(); + } metadata_.fragmentationSize()[pid][static_cast(cid)] = - (*stats_.fragmentationSize)[pid][cid].get(); + fragmentationSize; + } if (isCompactCachePool_[pid]) { metadata_.compactCachePools()->push_back(pid); @@ -3854,8 +3931,19 @@ void CacheAllocator::initStats() { // deserialize the fragmentation size of each thread. for (const auto& pid : *metadata_.fragmentationSize()) { for (const auto& cid : pid.second) { - (*stats_.fragmentationSize)[pid.first][cid.first].set( - static_cast(cid.second)); + //in multi-tier we serialized as the sum - no way + //to get back so just divide the two for now + //TODO: proper multi-tier serialization + uint64_t total = static_cast(cid.second); + uint64_t part = total / getNumTiers(); + uint64_t sum = 0; + for (TierId tid = 1; tid < getNumTiers(); tid++) { + (*stats_.fragmentationSize)[tid][pid.first][cid.first].set(part); + sum += part; + } + uint64_t leftover = total - sum; + (*stats_.fragmentationSize)[0][pid.first][cid.first].set(leftover); + } } diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 1de39104ea..edf23a3191 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1240,6 +1240,8 @@ class CacheAllocator : public CacheBase { // pool stats by pool id PoolStats getPoolStats(PoolId pid) const override final; + // pool stats by tier id and pool id + PoolStats getPoolStats(TierId tid, PoolId pid) const; // This can be expensive so it is not part of PoolStats PoolEvictionAgeStats getPoolEvictionAgeStats( @@ -2045,9 +2047,9 @@ auto& mmContainer = getMMContainer(tid, pid, cid); XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); if (candidate->hasChainedItem()) { - (*stats_.chainedItemEvictions)[pid][cid].inc(); + (*stats_.chainedItemEvictions)[tid][pid][cid].inc(); } else { - (*stats_.regularItemEvictions)[pid][cid].inc(); + (*stats_.regularItemEvictions)[tid][pid][cid].inc(); } // it's safe to recycle the item here as there are no more diff --git a/cachelib/allocator/CacheStats.cpp b/cachelib/allocator/CacheStats.cpp index 55305c7903..a5dd962dc2 100644 --- a/cachelib/allocator/CacheStats.cpp +++ b/cachelib/allocator/CacheStats.cpp @@ -23,18 +23,21 @@ namespace cachelib { namespace detail { void Stats::init() { - cacheHits = std::make_unique(); - allocAttempts = std::make_unique(); - evictionAttempts = std::make_unique(); - fragmentationSize = std::make_unique(); - allocFailures = std::make_unique(); - chainedItemEvictions = std::make_unique(); - regularItemEvictions = std::make_unique(); + cacheHits = std::make_unique(); + allocAttempts = std::make_unique(); + evictionAttempts = std::make_unique(); + fragmentationSize = std::make_unique(); + allocFailures = std::make_unique(); + chainedItemEvictions = std::make_unique(); + regularItemEvictions = std::make_unique(); + numWritebacks = std::make_unique(); auto initToZero = [](auto& a) { - for (auto& s : a) { - for (auto& c : s) { + for (auto& t : a) { + for (auto& p : t) { + for (auto& c : p) { c.set(0); } + } } }; @@ -44,6 +47,7 @@ void Stats::init() { initToZero(*fragmentationSize); initToZero(*chainedItemEvictions); initToZero(*regularItemEvictions); + initToZero(*numWritebacks); classAllocLatency = std::make_unique(); } @@ -116,20 +120,43 @@ void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { ret.nvmEvictionSecondsToExpiry = this->nvmEvictionSecondsToExpiry_.estimate(); ret.nvmPutSize = this->nvmPutSize_.estimate(); - auto accum = [](const PerPoolClassAtomicCounters& c) { - uint64_t sum = 0; - for (const auto& x : c) { - for (const auto& v : x) { - sum += v.get(); - } + auto accum = [](const PerTierPerPoolClassAtomicCounters& t) { + std::vector stat; + for (const auto& c : t) { + uint64_t sum = 0; + for (const auto& x : c) { + for (const auto& v : x) { + sum += v.get(); + } + } + stat.push_back(sum); + } + return stat; + }; + + auto accumTL = [](const PerTierPerPoolClassTLCounters& t) { + std::vector stat; + for (const auto& c : t) { + uint64_t sum = 0; + for (const auto& x : c) { + for (const auto& v : x) { + sum += v.get(); + } + } + stat.push_back(sum); } - return sum; + return stat; }; ret.allocAttempts = accum(*allocAttempts); ret.evictionAttempts = accum(*evictionAttempts); ret.allocFailures = accum(*allocFailures); - ret.numEvictions = accum(*chainedItemEvictions); - ret.numEvictions += accum(*regularItemEvictions); + auto chainedEvictions = accum(*chainedItemEvictions); + auto regularEvictions = accum(*regularItemEvictions); + for (TierId tid = 0; tid < chainedEvictions.size(); tid++) { + ret.numEvictions.push_back(chainedEvictions[tid] + regularEvictions[tid]); + } + ret.numWritebacks = accum(*numWritebacks); + ret.numCacheHits = accumTL(*cacheHits); ret.invalidAllocs = invalidAllocs.get(); ret.numRefcountOverflow = numRefcountOverflow.get(); @@ -145,6 +172,18 @@ void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { } // namespace detail +MMContainerStat& MMContainerStat::operator+=(const MMContainerStat& other) { + + size += other.size; + oldestTimeSec = std::min(oldestTimeSec,other.oldestTimeSec); + lruRefreshTime = std::max(lruRefreshTime,other.lruRefreshTime); + numHotAccesses += other.numHotAccesses; + numColdAccesses += other.numColdAccesses; + numWarmAccesses += other.numWarmAccesses; + numTailAccesses += other.numTailAccesses; + return *this; +} + PoolStats& PoolStats::operator+=(const PoolStats& other) { auto verify = [](bool isCompatible) { if (!isCompatible) { @@ -182,6 +221,7 @@ PoolStats& PoolStats::operator+=(const PoolStats& other) { d.allocFailures += s.allocFailures; d.fragmentationSize += s.fragmentationSize; d.numHits += s.numHits; + d.numWritebacks += s.numWritebacks; d.chainedItemEvictions += s.chainedItemEvictions; d.regularItemEvictions += s.regularItemEvictions; } @@ -237,6 +277,14 @@ uint64_t PoolStats::numEvictions() const noexcept { return n; } +uint64_t PoolStats::numWritebacks() const noexcept { + uint64_t n = 0; + for (const auto& s : cacheStats) { + n += s.second.numWritebacks; + } + return n; +} + uint64_t PoolStats::numItems() const noexcept { uint64_t n = 0; for (const auto& s : cacheStats) { @@ -245,6 +293,14 @@ uint64_t PoolStats::numItems() const noexcept { return n; } +uint64_t PoolStats::numHits() const noexcept { + uint64_t n = 0; + for (const auto& s : cacheStats) { + n += s.second.numHits; + } + return n; +} + uint64_t PoolStats::numAllocFailures() const { uint64_t n = 0; for (const auto& s : cacheStats) { diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index 11b5cf7596..48ea90f6b8 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -78,22 +78,25 @@ struct PoolEvictionAgeStats { // Stats for MM container struct MMContainerStat { // number of elements in the container. - size_t size; + size_t size{0}; // what is the unix timestamp in seconds of the oldest element existing in // the container. - uint64_t oldestTimeSec; + uint64_t oldestTimeSec{0}; // refresh time for LRU - uint64_t lruRefreshTime; + uint64_t lruRefreshTime{0}; // TODO: Make the MMContainerStat generic by moving the Lru/2Q specific // stats inside MMType and exporting them through a generic stats interface. // number of hits in each lru. - uint64_t numHotAccesses; - uint64_t numColdAccesses; - uint64_t numWarmAccesses; - uint64_t numTailAccesses; + uint64_t numHotAccesses{0}; + uint64_t numColdAccesses{0}; + uint64_t numWarmAccesses{0}; + uint64_t numTailAccesses{0}; + + // aggregate stats together (accross tiers) + MMContainerStat& operator+=(const MMContainerStat& other); }; // cache related stats for a given allocation class. @@ -114,13 +117,16 @@ struct CacheStat { uint64_t fragmentationSize{0}; // number of hits for this container. - uint64_t numHits; + uint64_t numHits{0}; // number of evictions from this class id that was of a chained item - uint64_t chainedItemEvictions; + uint64_t chainedItemEvictions{0}; // number of regular items that were evicted from this classId - uint64_t regularItemEvictions; + uint64_t regularItemEvictions{0}; + + // number of items that are moved to next tier + uint64_t numWritebacks{0}; // the stats from the mm container MMContainerStat containerStat; @@ -197,12 +203,18 @@ struct PoolStats { // number of evictions for this pool uint64_t numEvictions() const noexcept; + // number of writebacks for this pool + uint64_t numWritebacks() const noexcept; + // number of all items in this pool uint64_t numItems() const noexcept; // total number of allocations currently in this pool uint64_t numActiveAllocs() const noexcept; + // number of hits for an alloc class in this pool + uint64_t numHits() const noexcept; + // number of hits for an alloc class in this pool uint64_t numHitsForClass(ClassId cid) const { return cacheStats.at(cid).numHits; @@ -452,16 +464,22 @@ struct GlobalCacheStats { uint64_t numNvmItemRemovedSetSize{0}; // number of attempts to allocate an item - uint64_t allocAttempts{0}; + std::vector allocAttempts; // number of eviction attempts - uint64_t evictionAttempts{0}; + std::vector evictionAttempts; // number of failures to allocate an item due to internal error - uint64_t allocFailures{0}; + std::vector allocFailures; // number of evictions across all the pools in the cache. - uint64_t numEvictions{0}; + std::vector numEvictions; + + // number of writebacks across all the pools in the cache. + std::vector numWritebacks; + + // number of hits per tier across all the pools in the cache. + std::vector numCacheHits; // number of allocation attempts with invalid input params. uint64_t invalidAllocs{0}; diff --git a/cachelib/allocator/CacheStatsInternal.h b/cachelib/allocator/CacheStatsInternal.h index 19a15fbbd4..ef41ea7bbc 100644 --- a/cachelib/allocator/CacheStatsInternal.h +++ b/cachelib/allocator/CacheStatsInternal.h @@ -212,23 +212,26 @@ struct Stats { // we're currently writing into flash. mutable util::PercentileStats nvmPutSize_; - using PerPoolClassAtomicCounters = + using PerTierPerPoolClassAtomicCounters = std::array< std::array, - MemoryPoolManager::kMaxPools>; + MemoryPoolManager::kMaxPools>, + CacheBase::kMaxTiers>; // count of a stat for a specific allocation class - using PerPoolClassTLCounters = + using PerTierPerPoolClassTLCounters = std::array< std::array, - MemoryPoolManager::kMaxPools>; + MemoryPoolManager::kMaxPools>, + CacheBase::kMaxTiers>; // hit count for every alloc class in every pool - std::unique_ptr cacheHits{}; - std::unique_ptr allocAttempts{}; - std::unique_ptr evictionAttempts{}; - std::unique_ptr allocFailures{}; - std::unique_ptr fragmentationSize{}; - std::unique_ptr chainedItemEvictions{}; - std::unique_ptr regularItemEvictions{}; + std::unique_ptr cacheHits{}; + std::unique_ptr allocAttempts{}; + std::unique_ptr evictionAttempts{}; + std::unique_ptr allocFailures{}; + std::unique_ptr fragmentationSize{}; + std::unique_ptr chainedItemEvictions{}; + std::unique_ptr regularItemEvictions{}; + std::unique_ptr numWritebacks{}; using PerTierPoolClassRollingStats = std::array< std::array, diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp index 1db7fce112..a08ee04e6d 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp @@ -25,6 +25,7 @@ using LruAllocatorMemoryTiersTest = AllocatorMemoryTiersTest; // TODO(MEMORY_TIER): add more tests with different eviction policies TEST_F(LruAllocatorMemoryTiersTest, MultiTiersInvalid) { this->testMultiTiersInvalid(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid(); } +TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidStats) { this->testMultiTiersValidStats(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersBackgroundMovers ) { this->testMultiTiersBackgroundMovers(); } TEST_F(LruAllocatorMemoryTiersTest, MultiTiersRemoveDuringEviction) { this->testMultiTiersRemoveDuringEviction(); } diff --git a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h index 616de6e9db..5da2d01b5d 100644 --- a/cachelib/allocator/tests/AllocatorMemoryTiersTest.h +++ b/cachelib/allocator/tests/AllocatorMemoryTiersTest.h @@ -93,6 +93,62 @@ class AllocatorMemoryTiersTest : public AllocatorTest { ASSERT_NO_THROW(alloc->insertOrReplace(handle)); } + void testMultiTiersValidStats() { + typename AllocatorT::Config config; + size_t nSlabs = 20; + config.setCacheSize(nSlabs * Slab::kSize); + config.enableCachePersistence("/tmp"); + ASSERT_NO_THROW(config.configureMemoryTiers( + {MemoryTierCacheConfig::fromShm().setRatio(1).setMemBind( + std::string("0")), + MemoryTierCacheConfig::fromShm().setRatio(2).setMemBind( + std::string("0"))})); + + auto alloc = std::make_unique(AllocatorT::SharedMemNew, config); + ASSERT(alloc != nullptr); + size_t keyLen = 8; + auto pool = alloc->addPool("default", alloc->getCacheMemoryStats().ramCacheSize); + std::vector valsize = {1000}; + std::vector itemCount; + std::vector evictCount; + for (uint32_t tid = 0; tid < 2; tid++) { + this->fillUpPoolUntilEvictions(*alloc, tid, pool, valsize, keyLen); + auto stats = alloc->getPoolStats(tid, pool); + const auto& classIds = stats.mpStats.classIds; + uint32_t prev = 0; + ClassId cid = 0; + for (const ClassId c : classIds) { + uint32_t currSize = stats.cacheStats[c].allocSize; + if (prev <= valsize[0] && valsize[0] <= currSize) { + cid = c; + break; + } + prev = currSize; + } + + std::cout << "Tid: " << tid << " cid: " << static_cast(cid) + << " items: " << stats.cacheStats[cid].numItems() + << " evicts: " << stats.cacheStats[cid].numEvictions() + << std::endl; + ASSERT_GE(stats.cacheStats[cid].numItems(), 1); + ASSERT_EQ(stats.cacheStats[cid].numEvictions(), 1); + itemCount.push_back(stats.cacheStats[cid].numItems()); + evictCount.push_back(stats.cacheStats[cid].numEvictions()); + //first tier should have some writebacks to second tier + //second tier should not have any writebacks since it + //is last memory tier + if (tid == 0) { + ASSERT_EQ(stats.cacheStats[cid].numWritebacks, 1); + } else { + ASSERT_EQ(0, stats.cacheStats[cid].numWritebacks); + } + } + for (uint32_t tid = 1; tid < 2; tid++) { + ASSERT_NE(itemCount[tid],itemCount[tid-1]); + ASSERT_EQ(evictCount[tid],evictCount[tid-1]); + } + } + void testMultiTiersBackgroundMovers() { typename AllocatorT::Config config; config.setCacheSize(10 * Slab::kSize); diff --git a/cachelib/allocator/tests/TestBase-inl.h b/cachelib/allocator/tests/TestBase-inl.h index 4d45e981bc..79dc6f44be 100644 --- a/cachelib/allocator/tests/TestBase-inl.h +++ b/cachelib/allocator/tests/TestBase-inl.h @@ -98,6 +98,30 @@ void AllocatorTest::fillUpPoolUntilEvictions( } while (allocs != 0); } +template +void AllocatorTest::fillUpPoolUntilEvictions( + AllocatorT& alloc, + TierId tid, + PoolId poolId, + const std::vector& sizes, + unsigned int keyLen) { + unsigned int allocs = 0; + do { + allocs = 0; + for (const auto size : sizes) { + const auto key = getRandomNewKey(alloc, keyLen); + ASSERT_EQ(alloc.find(key), nullptr); + const size_t prev = alloc.getPoolByTid(poolId, tid).getCurrentAllocSize(); + auto handle = util::allocateAccessible(alloc, poolId, key, size); + if (handle && prev != alloc.getPoolByTid(poolId, tid).getCurrentAllocSize()) { + // this means we did not cause an eviction. + ASSERT_GE(handle->getSize(), size); + allocs++; + } + } + } while (allocs != 0); +} + template void AllocatorTest::testAllocWithoutEviction( AllocatorT& alloc, diff --git a/cachelib/allocator/tests/TestBase.h b/cachelib/allocator/tests/TestBase.h index 54032e3257..858da2bb95 100644 --- a/cachelib/allocator/tests/TestBase.h +++ b/cachelib/allocator/tests/TestBase.h @@ -69,6 +69,11 @@ class AllocatorTest : public SlabAllocatorTestBase { PoolId pid, const std::vector& sizes, unsigned int keyLen); + void fillUpPoolUntilEvictions(AllocatorT& alloc, + TierId tid, + PoolId pid, + const std::vector& sizes, + unsigned int keyLen); void fillUpOneSlab(AllocatorT& alloc, PoolId poolId, const uint32_t size, diff --git a/cachelib/cachebench/cache/Cache-inl.h b/cachelib/cachebench/cache/Cache-inl.h index 3f54fd31e4..36a69fe7a9 100644 --- a/cachelib/cachebench/cache/Cache-inl.h +++ b/cachelib/cachebench/cache/Cache-inl.h @@ -645,18 +645,25 @@ double Cache::getNvmBytesWritten() const { template Stats Cache::getStats() const { - PoolStats aggregate = cache_->getPoolStats(pools_[0]); - auto usageFraction = - 1.0 - (static_cast(aggregate.freeMemoryBytes())) / - aggregate.poolUsableSize; + Stats ret; - ret.poolUsageFraction.push_back(usageFraction); - for (size_t pid = 1; pid < pools_.size(); pid++) { - auto poolStats = cache_->getPoolStats(static_cast(pid)); - usageFraction = 1.0 - (static_cast(poolStats.freeMemoryBytes())) / - poolStats.poolUsableSize; - ret.poolUsageFraction.push_back(usageFraction); - aggregate += poolStats; + for (TierId tid = 0; tid < cache_->getNumTiers(); tid++) { + PoolStats aggregate = cache_->getPoolStats(tid,pools_[0]); + auto usageFraction = + 1.0 - (static_cast(aggregate.freeMemoryBytes())) / + aggregate.poolUsableSize; + ret.poolUsageFraction[tid].push_back(usageFraction); + for (size_t pid = 1; pid < pools_.size(); pid++) { + auto poolStats = cache_->getPoolStats(tid, static_cast(pid)); + usageFraction = 1.0 - (static_cast(poolStats.freeMemoryBytes())) / + poolStats.poolUsableSize; + ret.poolUsageFraction[tid].push_back(usageFraction); + aggregate += poolStats; + } + ret.numEvictions.push_back(aggregate.numEvictions()); + ret.numWritebacks.push_back(aggregate.numWritebacks()); + ret.numCacheHits.push_back(aggregate.numHits()); + ret.numItems.push_back(aggregate.numItems()); } std::map>> allocationClassStats{}; @@ -691,8 +698,6 @@ Stats Cache::getStats() const { ret.backgndPromoStats.nTraversals = cacheStats.promotionStats.runCount; - ret.numEvictions = aggregate.numEvictions(); - ret.numItems = aggregate.numItems(); ret.evictAttempts = cacheStats.evictionAttempts; ret.allocAttempts = cacheStats.allocAttempts; ret.allocFailures = cacheStats.allocFailures; diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index 706dfdf67a..eecb89bb96 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -52,15 +52,18 @@ struct BackgroundPromotionStats { struct Stats { BackgroundEvictionStats backgndEvicStats; BackgroundPromotionStats backgndPromoStats; + ReaperStats reaperStats; - uint64_t numEvictions{0}; - uint64_t numItems{0}; + std::vector numEvictions; + std::vector numWritebacks; + std::vector numCacheHits; + std::vector numItems; - uint64_t evictAttempts{0}; - uint64_t allocAttempts{0}; - uint64_t allocFailures{0}; + std::vector evictAttempts{0}; + std::vector allocAttempts{0}; + std::vector allocFailures{0}; - std::vector poolUsageFraction; + std::map> poolUsageFraction; uint64_t numCacheGets{0}; uint64_t numCacheGetMiss{0}; @@ -143,19 +146,31 @@ struct Stats { void render(std::ostream& out) const { auto totalMisses = getTotalMisses(); const double overallHitRatio = invertPctFn(totalMisses, numCacheGets); - out << folly::sformat("Items in RAM : {:,}", numItems) << std::endl; - out << folly::sformat("Items in NVM : {:,}", numNvmItems) << std::endl; - - out << folly::sformat("Alloc Attempts: {:,} Success: {:.2f}%", - allocAttempts, - invertPctFn(allocFailures, allocAttempts)) - << std::endl; - out << folly::sformat("Evict Attempts: {:,} Success: {:.2f}%", - evictAttempts, - pctFn(numEvictions, evictAttempts)) - << std::endl; - out << folly::sformat("RAM Evictions : {:,}", numEvictions) << std::endl; - + const auto nTiers = numItems.size(); + for (TierId tid = 0; tid < nTiers; tid++) { + out << folly::sformat("Items in Tier {} : {:,}", tid, numItems[tid]) << std::endl; + } + out << folly::sformat("Items in NVM : {:,}", numNvmItems) << std::endl; + for (TierId tid = 0; tid < nTiers; tid++) { + out << folly::sformat("Tier {} Alloc Attempts: {:,} Success: {:.2f}%", + tid, + allocAttempts[tid], + invertPctFn(allocFailures[tid], allocAttempts[tid])) + << std::endl; + } + for (TierId tid = 0; tid < nTiers; tid++) { + out << folly::sformat( + "Tier {} Evict Attempts: {:,} Success: {:.2f}%", + tid, + evictAttempts[tid], + pctFn(numEvictions[tid], evictAttempts[tid])) + << std::endl; + } + for (TierId tid = 0; tid < nTiers; tid++) { + out << folly::sformat("Tier {} Evictions : {:,} Writebacks: {:,} Success: {:.2f}%", + tid, numEvictions[tid], numWritebacks[tid], + invertPctFn(numEvictions[tid] - numWritebacks[tid], numEvictions[tid])) << std::endl; + } auto foreachAC = [&](auto &map, auto cb) { for (auto &tidStats : map) { for (auto &pidStat : tidStats.second) { @@ -166,10 +181,16 @@ struct Stats { } }; - for (auto pid = 0U; pid < poolUsageFraction.size(); pid++) { - out << folly::sformat("Fraction of pool {:,} used : {:.2f}", pid, - poolUsageFraction[pid]) - << std::endl; + for (auto entry : poolUsageFraction) { + auto tid = entry.first; + auto usageFraction = entry.second; + for (auto pid = 0U; pid < usageFraction.size(); pid++) { + out << folly::sformat("Tier {} fraction of pool {:,} used : {:.2f}", + tid, + pid, + usageFraction[pid]) + << std::endl; + } } if (FLAGS_report_ac_memory_usage_stats != "") { @@ -211,8 +232,8 @@ struct Stats { // If the pool is not full, extrapolate usageFraction for AC assuming it // will grow at the same rate. This value will be the same for all ACs. - auto acUsageFraction = (poolUsageFraction[pid] < 1.0) - ? poolUsageFraction[pid] + const auto acUsageFraction = (poolUsageFraction.at(tid)[pid] < 1.0) + ? poolUsageFraction.at(tid)[pid] : stats.usageFraction(); out << folly::sformat( @@ -234,7 +255,11 @@ struct Stats { out << folly::sformat("Cache Gets : {:,}", numCacheGets) << std::endl; out << folly::sformat("Hit Ratio : {:6.2f}%", overallHitRatio) << std::endl; - + for (TierId tid = 0; tid < numCacheHits.size(); tid++) { + double tierHitRatio = pctFn(numCacheHits[tid],numCacheGets); + out << folly::sformat("Tier {} Hit Ratio : {:6.2f}%", tid, tierHitRatio) + << std::endl; + } if (FLAGS_report_api_latency) { auto printLatencies = [&out](folly::StringPiece cat, @@ -276,6 +301,14 @@ struct Stats { }); } + if (reaperStats.numReapedItems > 0) { + + out << folly::sformat("Reaper reaped: {:,} visited: {:,} traversals: {:,} avg traversal time: {:,}", + reaperStats.numReapedItems,reaperStats.numVisitedItems, + reaperStats.numTraversals,reaperStats.avgTraversalTimeMs) + << std::endl; + } + if (numNvmGets > 0 || numNvmDeletes > 0 || numNvmPuts > 0) { const double ramHitRatio = invertPctFn(numCacheGetMiss, numCacheGets); const double nvmHitRatio = invertPctFn(numNvmGetMiss, numNvmGets); @@ -411,8 +444,8 @@ struct Stats { } if (numCacheEvictions > 0) { - out << folly::sformat("Total eviction executed {}", numCacheEvictions) - << std::endl; + out << folly::sformat("Total evictions executed {:,}", numCacheEvictions) + << std::endl; } } @@ -470,7 +503,8 @@ struct Stats { }; auto totalMisses = getTotalMisses(); - counters["num_items"] = numItems; + //TODO: per tier + counters["num_items"] = std::accumulate(numItems.begin(),numItems.end(),0); counters["num_nvm_items"] = numNvmItems; counters["hit_rate"] = calcInvertPctFn(totalMisses, numCacheGets); From aedaf975f2c25cea964031549e487f6c61799fcb Mon Sep 17 00:00:00 2001 From: Sounak Gupta Date: Tue, 28 Mar 2023 12:11:15 -0700 Subject: [PATCH 19/31] dummy change to trigger container image rebuild --- docker/images/install-dsa-deps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/images/install-dsa-deps.sh b/docker/images/install-dsa-deps.sh index b4c62ecc93..265011dd70 100755 --- a/docker/images/install-dsa-deps.sh +++ b/docker/images/install-dsa-deps.sh @@ -15,6 +15,7 @@ rm -rf idxd-config # Install DML Library git clone --recursive https://github.com/intel/DML.git cd DML +git checkout e44443c24d53552b248b9869b1b16f89cd970f52 mkdir build cd build cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo .. From 1f21fcedce09330f07993c1c15cfb4bf1399fd4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Chor=C4=85=C5=BCewicz?= Date: Thu, 27 Apr 2023 13:23:07 -0700 Subject: [PATCH 20/31] Fix token creation and stats (#79) * Fix issue with token creation * Do not increment evictFail* stats if evictFailConcurrentFill were incremented --- cachelib/allocator/CacheAllocator-inl.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index f36fbd57de..013139693e 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -1600,14 +1600,15 @@ CacheAllocator::getNextCandidate(TierId tid, ? &toRecycle_->asChainedItem().getParentItem(compressor_) : toRecycle_; - typename NvmCacheT::PutToken putToken; - if (lastTier) { - // if it's last tier, the item will be evicted - // need to create put token before marking it exclusive - putToken = createPutToken(*candidate_); - } + // if it's last tier, the item will be evicted + // need to create put token before marking it exclusive + const bool evictToNvmCache = lastTier && shouldWriteToNvmCache(*candidate_); + + auto token_ = evictToNvmCache + ? nvmCache_->createPutToken(candidate_->getKey()) + : typename NvmCacheT::PutToken{}; - if (lastTier && shouldWriteToNvmCache(*candidate_) && !putToken.isValid()) { + if (evictToNvmCache && !token_.isValid()) { stats_.evictFailConcurrentFill.inc(); ++itr; continue; @@ -1630,7 +1631,7 @@ CacheAllocator::getNextCandidate(TierId tid, // since we won't be moving the item to the next tier toRecycle = toRecycle_; candidate = candidate_; - token = std::move(putToken); + token = std::move(token_); // Check if parent changed for chained items - if yes, we cannot // remove the child from the mmContainer as we will not be evicting From 9e27d35a7adf90b906ad28fef8c9e3429e26a61f Mon Sep 17 00:00:00 2001 From: Sounak Gupta Date: Tue, 9 May 2023 07:16:17 -0700 Subject: [PATCH 21/31] Updated the docker gcc version to 12 (#83) updated the docker gcc version to 12 --------- Co-authored-by: Matt Rae --- docker/images/centos-8streams.Dockerfile | 4 ++++ docker/run-build.sh | 3 +++ 2 files changed, 7 insertions(+) diff --git a/docker/images/centos-8streams.Dockerfile b/docker/images/centos-8streams.Dockerfile index 29752c5d98..b916ab760c 100644 --- a/docker/images/centos-8streams.Dockerfile +++ b/docker/images/centos-8streams.Dockerfile @@ -17,6 +17,10 @@ json-c-devel \ perf \ numactl +RUN dnf -y install gcc-toolset-12 +RUN echo "source /opt/rh/gcc-toolset-12/enable" >> /etc/bashrc +SHELL ["/bin/bash", "--login", "-c"] + COPY ./install-cachelib-deps.sh ./install-cachelib-deps.sh RUN ./install-cachelib-deps.sh diff --git a/docker/run-build.sh b/docker/run-build.sh index 02c7caf731..bc04819f18 100755 --- a/docker/run-build.sh +++ b/docker/run-build.sh @@ -11,6 +11,9 @@ function sudo_password() { cd .. mkdir build cd build + +source /opt/rh/gcc-toolset-12/enable + cmake ../cachelib -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=/opt -DCMAKE_BUILD_TYPE=Debug sudo_password make install -j$(nproc) From da7a6bb973e6df00bb294f25532cecc5c0237b57 Mon Sep 17 00:00:00 2001 From: Sergei Vinogradov Date: Wed, 17 May 2023 13:36:42 +0200 Subject: [PATCH 22/31] NUMA bindigs support for private memory (#82) --- cachelib/allocator/CMakeLists.txt | 1 + cachelib/allocator/CacheAllocator-inl.h | 42 ++++++++---- cachelib/allocator/CacheAllocator.h | 7 +- cachelib/allocator/MemoryTierCacheConfig.h | 9 ++- cachelib/allocator/PrivateMemoryManager.cpp | 50 ++++++++++++++ cachelib/allocator/PrivateMemoryManager.h | 44 +++++++++++++ cachelib/cachebench/util/CacheConfig.h | 2 +- cachelib/common/CMakeLists.txt | 1 + cachelib/common/Utils.cpp | 17 +++++ cachelib/common/Utils.h | 72 +++++++++++++++++++++ cachelib/shm/PosixShmSegment.cpp | 2 + cachelib/shm/ShmCommon.h | 57 +--------------- cachelib/shm/SysVShmSegment.cpp | 17 +---- examples/single_tier_cache/main.cpp | 2 +- 14 files changed, 236 insertions(+), 87 deletions(-) create mode 100644 cachelib/allocator/PrivateMemoryManager.cpp create mode 100644 cachelib/allocator/PrivateMemoryManager.h diff --git a/cachelib/allocator/CMakeLists.txt b/cachelib/allocator/CMakeLists.txt index 6103cdc823..0f96a0cd7f 100644 --- a/cachelib/allocator/CMakeLists.txt +++ b/cachelib/allocator/CMakeLists.txt @@ -55,6 +55,7 @@ add_library (cachelib_allocator PoolOptimizeStrategy.cpp PoolRebalancer.cpp PoolResizer.cpp + PrivateMemoryManager.cpp RebalanceStrategy.cpp SlabReleaseStats.cpp TempShmMapping.cpp diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 013139693e..3c904332a0 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -59,6 +59,9 @@ CacheAllocator::CacheAllocator( tempShm_(type == InitMemType::kNone && isOnShm_ ? std::make_unique(config_.getCacheSize()) : nullptr), + privMemManager_(type == InitMemType::kNone && !isOnShm_ + ? std::make_unique() + : nullptr), shmManager_(type != InitMemType::kNone ? std::make_unique(config_.cacheDir, config_.isUsingPosixShm()) @@ -121,6 +124,16 @@ ShmSegmentOpts CacheAllocator::createShmCacheOpts(TierId tid) { return opts; } +template +PrivateSegmentOpts CacheAllocator::createPrivateSegmentOpts(TierId tid) { + PrivateSegmentOpts opts; + opts.alignment = sizeof(Slab); + auto memoryTierConfigs = config_.getMemoryTierConfigs(); + opts.memBindNumaNodes = memoryTierConfigs[tid].getMemBind(); + + return opts; +} + template size_t CacheAllocator::memoryTierSize(TierId tid) const { auto& memoryTierConfigs = config_.memoryTierConfigs; @@ -133,21 +146,18 @@ size_t CacheAllocator::memoryTierSize(TierId tid) const { } template -std::vector> -CacheAllocator::createPrivateAllocator() { - std::vector> allocators; - +std::unique_ptr +CacheAllocator::createPrivateAllocator(TierId tid) { if (isOnShm_) - allocators.emplace_back(std::make_unique( + return std::make_unique( getAllocatorConfig(config_), tempShm_->getAddr(), - config_.getCacheSize())); + memoryTierSize(tid)); else - allocators.emplace_back(std::make_unique( + return std::make_unique( getAllocatorConfig(config_), - config_.getCacheSize())); - - return allocators; + privMemManager_->createMapping(config_.size, createPrivateSegmentOpts(tid)), + memoryTierSize(tid)); } template @@ -176,6 +186,16 @@ CacheAllocator::restoreMemoryAllocator(TierId tid) { config_.disableFullCoredump); } +template +std::vector> +CacheAllocator::createPrivateAllocators() { + std::vector> allocators; + for (int tid = 0; tid < getNumTiers(); tid++) { + allocators.emplace_back(createPrivateAllocator(tid)); + } + return allocators; +} + template std::vector> CacheAllocator::createAllocators() { @@ -309,7 +329,7 @@ std::vector> CacheAllocator::initAllocator( InitMemType type) { if (type == InitMemType::kNone) { - return createPrivateAllocator(); + return createPrivateAllocators(); } else if (type == InitMemType::kMemNew) { return createAllocators(); } else if (type == InitMemType::kMemAttach) { diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index edf23a3191..e571da06e6 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -60,6 +60,7 @@ #include "cachelib/allocator/PoolOptimizer.h" #include "cachelib/allocator/PoolRebalancer.h" #include "cachelib/allocator/PoolResizer.h" +#include "cachelib/allocator/PrivateMemoryManager.h" #include "cachelib/allocator/ReadOnlySharedCacheView.h" #include "cachelib/allocator/Reaper.h" #include "cachelib/allocator/RebalanceStrategy.h" @@ -2185,6 +2186,8 @@ auto& mmContainer = getMMContainer(tid, pid, cid); std::chrono::seconds timeout = std::chrono::seconds{0}); ShmSegmentOpts createShmCacheOpts(TierId tid); + PrivateSegmentOpts createPrivateSegmentOpts(TierId tid); + std::unique_ptr createPrivateAllocator(TierId tid); std::unique_ptr createNewMemoryAllocator(TierId tid); std::unique_ptr restoreMemoryAllocator(TierId tid); std::unique_ptr restoreCCacheManager(TierId tid); @@ -2234,7 +2237,7 @@ auto& mmContainer = getMMContainer(tid, pid, cid); // @throw std::runtime_error if type is invalid std::vector> initAllocator(InitMemType type); - std::vector> createPrivateAllocator(); + std::vector> createPrivateAllocators(); std::vector> createAllocators(); std::vector> restoreAllocators(); @@ -2401,6 +2404,8 @@ auto& mmContainer = getMMContainer(tid, pid, cid); // is not persisted when cache process exits. std::unique_ptr tempShm_; + std::unique_ptr privMemManager_; + std::unique_ptr shmManager_; // Deserialize data to restore cache allocator. Used only while attaching to diff --git a/cachelib/allocator/MemoryTierCacheConfig.h b/cachelib/allocator/MemoryTierCacheConfig.h index 1b9477c048..ee579a5386 100644 --- a/cachelib/allocator/MemoryTierCacheConfig.h +++ b/cachelib/allocator/MemoryTierCacheConfig.h @@ -16,11 +16,14 @@ #pragma once +#include "cachelib/common/Utils.h" #include "cachelib/shm/ShmCommon.h" namespace facebook { namespace cachelib { class MemoryTierCacheConfig { + using bitmask_type = util::NumaBitMask; + public: // Creates instance of MemoryTierCacheConfig for Posix/SysV Shared memory. static MemoryTierCacheConfig fromShm() { return MemoryTierCacheConfig(); } @@ -39,12 +42,12 @@ class MemoryTierCacheConfig { size_t getRatio() const noexcept { return ratio; } // Allocate memory only from specified NUMA nodes - MemoryTierCacheConfig& setMemBind(const NumaBitMask& _numaNodes) { + MemoryTierCacheConfig& setMemBind(const bitmask_type& _numaNodes) { numaNodes = _numaNodes; return *this; } - const NumaBitMask& getMemBind() const noexcept { return numaNodes; } + const bitmask_type& getMemBind() const noexcept { return numaNodes; } size_t calculateTierSize(size_t totalCacheSize, size_t partitionNum) const { // TODO: Call this method when tiers are enabled in allocator @@ -71,7 +74,7 @@ class MemoryTierCacheConfig { size_t ratio{1}; // Numa node(s) to bind the tier - NumaBitMask numaNodes; + bitmask_type numaNodes; // TODO: introduce a container for tier settings when adding support for // file-mapped memory diff --git a/cachelib/allocator/PrivateMemoryManager.cpp b/cachelib/allocator/PrivateMemoryManager.cpp new file mode 100644 index 0000000000..afcf1b2202 --- /dev/null +++ b/cachelib/allocator/PrivateMemoryManager.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cachelib/allocator/PrivateMemoryManager.h" + +#include + +namespace facebook { +namespace cachelib { + +PrivateMemoryManager::~PrivateMemoryManager() { + for (auto& entry : mappings) { + util::munmapMemory(entry.first, entry.second); + } +} + +void* PrivateMemoryManager::createMapping(size_t size, + PrivateSegmentOpts opts) { + void* addr = util::mmapAlignedZeroedMemory(opts.alignment, size); + auto guard = folly::makeGuard([&]() { + util::munmapMemory(addr, size); + mappings.erase(addr); + }); + + XDCHECK_EQ(reinterpret_cast(addr) & (opts.alignment - 1), 0ULL); + + if (!opts.memBindNumaNodes.empty()) { + util::mbindMemory(addr, size, MPOL_BIND, opts.memBindNumaNodes, 0); + } + + mappings.emplace(addr, size); + + guard.dismiss(); + return addr; +} +} // namespace cachelib +} // namespace facebook \ No newline at end of file diff --git a/cachelib/allocator/PrivateMemoryManager.h b/cachelib/allocator/PrivateMemoryManager.h new file mode 100644 index 0000000000..7880ca928a --- /dev/null +++ b/cachelib/allocator/PrivateMemoryManager.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "cachelib/common/Utils.h" + +namespace facebook { +namespace cachelib { + +struct PrivateSegmentOpts { + size_t alignment{1}; // alignment for mapping. + util::NumaBitMask memBindNumaNodes; +}; + +class PrivateMemoryManager { + public: + PrivateMemoryManager() {} + ~PrivateMemoryManager(); + + void* createMapping(size_t size, PrivateSegmentOpts opts); + + private: + std::unordered_map mappings; +}; + +} // namespace cachelib +} // namespace facebook \ No newline at end of file diff --git a/cachelib/cachebench/util/CacheConfig.h b/cachelib/cachebench/util/CacheConfig.h index 411fe73275..6481aa67c3 100644 --- a/cachelib/cachebench/util/CacheConfig.h +++ b/cachelib/cachebench/util/CacheConfig.h @@ -52,7 +52,7 @@ struct MemoryTierConfig : public JSONConfig { MemoryTierCacheConfig getMemoryTierCacheConfig() { MemoryTierCacheConfig config = MemoryTierCacheConfig::fromShm(); config.setRatio(ratio); - config.setMemBind(NumaBitMask(memBindNodes)); + config.setMemBind(util::NumaBitMask(memBindNodes)); return config; } diff --git a/cachelib/common/CMakeLists.txt b/cachelib/common/CMakeLists.txt index 1e6d1a887c..212f421324 100644 --- a/cachelib/common/CMakeLists.txt +++ b/cachelib/common/CMakeLists.txt @@ -39,6 +39,7 @@ target_link_libraries(cachelib_common PUBLIC Folly::folly_exception_tracer Folly::folly_exception_tracer_base Folly::folly_exception_counter + numa ) install(TARGETS cachelib_common diff --git a/cachelib/common/Utils.cpp b/cachelib/common/Utils.cpp index 82ec0bf72e..9b051519dc 100644 --- a/cachelib/common/Utils.cpp +++ b/cachelib/common/Utils.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -181,6 +182,22 @@ void* mmapAlignedZeroedMemory(size_t alignment, throw std::system_error(errno, std::system_category(), "Cannot mmap"); } +void munmapMemory(void* addr, size_t size) { munmap(addr, size); } + +void mbindMemory(void* addr, + unsigned long len, + int mode, + const NumaBitMask& mask, + unsigned int flags) { + auto nodesMask = mask.getNativeBitmask(); + + long ret = mbind(addr, len, mode, nodesMask->maskp, nodesMask->size, flags); + if (ret != 0) { + util::throwSystemError( + errno, folly::sformat("mbind() failed: {}", std::strerror(errno))); + } +} + void setMaxLockMemory(uint64_t bytes) { struct rlimit rlim { bytes, bytes diff --git a/cachelib/common/Utils.h b/cachelib/common/Utils.h index 4e4c839ef9..c94a445b4d 100644 --- a/cachelib/common/Utils.h +++ b/cachelib/common/Utils.h @@ -18,6 +18,8 @@ #include #include +#include +#include #include @@ -35,6 +37,57 @@ namespace facebook { namespace cachelib { namespace util { +class NumaBitMask { + public: + using native_bitmask_type = struct bitmask*; + + NumaBitMask() { nodesMask = numa_allocate_nodemask(); } + + NumaBitMask(const NumaBitMask& other) { + nodesMask = numa_allocate_nodemask(); + copy_bitmask_to_bitmask(other.nodesMask, nodesMask); + } + + NumaBitMask(NumaBitMask&& other) { + nodesMask = other.nodesMask; + other.nodesMask = nullptr; + } + + NumaBitMask(const std::string& str) { + nodesMask = numa_parse_nodestring_all(str.c_str()); + } + + ~NumaBitMask() { + if (nodesMask) { + numa_bitmask_free(nodesMask); + } + } + + constexpr NumaBitMask& operator=(const NumaBitMask& other) { + if (this != &other) { + if (!nodesMask) { + nodesMask = numa_allocate_nodemask(); + } + copy_bitmask_to_bitmask(other.nodesMask, nodesMask); + } + return *this; + } + + native_bitmask_type getNativeBitmask() const noexcept { return nodesMask; } + + NumaBitMask& setBit(unsigned int n) { + numa_bitmask_setbit(nodesMask, n); + return *this; + } + + bool empty() const noexcept { + return numa_bitmask_equal(numa_no_nodes_ptr, nodesMask) == 1; + } + + protected: + native_bitmask_type nodesMask = nullptr; +}; + // A wrapper class for functions to collect counters. // It can be initialized by either // 1. folly::StringPiece, double -> void, or @@ -295,6 +348,25 @@ void* mmapAlignedZeroedMemory(size_t alignment, size_t numBytes, bool noAccess = false); +// destroy the mapping created by mmapAlignedZeroedMemory +// +// @param addr the pointer to the memory to unmap +// @param size size of the memory region +void munmapMemory(void* addr, size_t size); + +// binds memory to the NUMA nodes specified by nmask. +// +// @param addr the pointer to the memory to bind. +// @param len length of the memory. +// @param mode mode supported by mmap call +// @param mask mask specifies node ids +// @param flags flags supported by mmap call +void mbindMemory(void* addr, + unsigned long len, + int mode, + const NumaBitMask& mask, + unsigned int flags); + // get the number of pages in the range which are resident in the process. // // @param mem memory start which is page aligned diff --git a/cachelib/shm/PosixShmSegment.cpp b/cachelib/shm/PosixShmSegment.cpp index 7d47d061d1..4c19e229fd 100644 --- a/cachelib/shm/PosixShmSegment.cpp +++ b/cachelib/shm/PosixShmSegment.cpp @@ -31,6 +31,8 @@ namespace facebook { namespace cachelib { +using NumaBitMask = util::NumaBitMask; + constexpr static mode_t kRWMode = 0666; typedef struct stat stat_t; diff --git a/cachelib/shm/ShmCommon.h b/cachelib/shm/ShmCommon.h index 8db8707515..bc451c46d1 100644 --- a/cachelib/shm/ShmCommon.h +++ b/cachelib/shm/ShmCommon.h @@ -15,8 +15,6 @@ */ #pragma once -#include -#include #include #include #include @@ -30,6 +28,8 @@ #include #pragma GCC diagnostic pop +#include "cachelib/common/Utils.h" + /* On Mac OS / FreeBSD, mmap(2) syscall does not support these flags */ #ifndef MAP_LOCKED #define MAP_LOCKED 0 @@ -72,62 +72,11 @@ enum PageSizeT { ONE_GB, }; -class NumaBitMask { - public: - using native_bitmask_type = struct bitmask*; - - NumaBitMask() { nodesMask = numa_allocate_nodemask(); } - - NumaBitMask(const NumaBitMask& other) { - nodesMask = numa_allocate_nodemask(); - copy_bitmask_to_bitmask(other.nodesMask, nodesMask); - } - - NumaBitMask(NumaBitMask&& other) { - nodesMask = other.nodesMask; - other.nodesMask = nullptr; - } - - NumaBitMask(const std::string& str) { - nodesMask = numa_parse_nodestring_all(str.c_str()); - } - - ~NumaBitMask() { - if (nodesMask) { - numa_bitmask_free(nodesMask); - } - } - - constexpr NumaBitMask& operator=(const NumaBitMask& other) { - if (this != &other) { - if (!nodesMask) { - nodesMask = numa_allocate_nodemask(); - } - copy_bitmask_to_bitmask(other.nodesMask, nodesMask); - } - return *this; - } - - native_bitmask_type getNativeBitmask() const noexcept { return nodesMask; } - - NumaBitMask& setBit(unsigned int n) { - numa_bitmask_setbit(nodesMask, n); - return *this; - } - - bool empty() const noexcept { - return numa_bitmask_equal(numa_no_nodes_ptr, nodesMask) == 1; - } - - protected: - native_bitmask_type nodesMask = nullptr; -}; - struct ShmSegmentOpts { PageSizeT pageSize{PageSizeT::NORMAL}; bool readOnly{false}; size_t alignment{1}; // alignment for mapping. - NumaBitMask memBindNumaNodes; + util::NumaBitMask memBindNumaNodes; explicit ShmSegmentOpts(PageSizeT p) : pageSize(p) {} explicit ShmSegmentOpts(PageSizeT p, bool ro) : pageSize(p), readOnly(ro) {} diff --git a/cachelib/shm/SysVShmSegment.cpp b/cachelib/shm/SysVShmSegment.cpp index 29485fa0c4..1cb28da70b 100644 --- a/cachelib/shm/SysVShmSegment.cpp +++ b/cachelib/shm/SysVShmSegment.cpp @@ -189,21 +189,6 @@ void shmCtlImpl(int shmid, int cmd, shmid_ds* buf) { } } -void mbindImpl(void* addr, - unsigned long len, - int mode, - - const NumaBitMask& memBindNumaNodes, - unsigned int flags) { - auto nodesMask = memBindNumaNodes.getNativeBitmask(); - - long ret = mbind(addr, len, mode, nodesMask->maskp, nodesMask->size, flags); - if (ret != 0) { - util::throwSystemError( - errno, folly::sformat("mbind() failed: {}", std::strerror(errno))); - } -} - } // namespace detail void ensureSizeforHugePage(size_t size) { @@ -300,7 +285,7 @@ void SysVShmSegment::memBind(void* addr) const { if (opts_.memBindNumaNodes.empty()) { return; } - detail::mbindImpl(addr, getSize(), MPOL_BIND, opts_.memBindNumaNodes, 0); + util::mbindMemory(addr, getSize(), MPOL_BIND, opts_.memBindNumaNodes, 0); } void SysVShmSegment::markForRemoval() { diff --git a/examples/single_tier_cache/main.cpp b/examples/single_tier_cache/main.cpp index de6373622c..9c19dfeea9 100644 --- a/examples/single_tier_cache/main.cpp +++ b/examples/single_tier_cache/main.cpp @@ -25,7 +25,7 @@ using CacheConfig = typename Cache::Config; using CacheKey = typename Cache::Key; using CacheReadHandle = typename Cache::ReadHandle; using MemoryTierCacheConfig = typename cachelib::MemoryTierCacheConfig; -using NumaBitMask = typename cachelib::NumaBitMask; +using NumaBitMask = typename cachelib::util::NumaBitMask; // Global cache object and a default cache pool std::unique_ptr gCache_; From b5ac4628ab03572a979277adeb2bd5c4cc8025bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Chor=C4=85=C5=BCewicz?= Date: Tue, 6 Jun 2023 09:05:29 -0700 Subject: [PATCH 23/31] Do not run cachelib-centos-8-5 on PRs (#85) --- .github/workflows/build-cachelib-centos-8-5.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-cachelib-centos-8-5.yml b/.github/workflows/build-cachelib-centos-8-5.yml index 14ab8cfa74..30247b2e94 100644 --- a/.github/workflows/build-cachelib-centos-8-5.yml +++ b/.github/workflows/build-cachelib-centos-8-5.yml @@ -13,8 +13,6 @@ # limitations under the License. name: build-cachelib-centos-8.5 on: -# push: - pull_request: schedule: - cron: '0 9 * * *' jobs: From 50d3ae5e17624f24ebdb97b01d148664843ed795 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Tue, 6 Jun 2023 12:37:48 -0400 Subject: [PATCH 24/31] correct handling for expired items in eviction (#86) - we first check if an item is expired under mmContainer lock and if so mark it for eviction so it is recycled back up to allocateInternalTier. --- cachelib/allocator/CacheAllocator-inl.h | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 3c904332a0..6a83559545 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -1596,12 +1596,13 @@ CacheAllocator::getNextCandidate(TierId tid, typename NvmCacheT::PutToken token; Item* toRecycle = nullptr; Item* candidate = nullptr; + bool isExpired = false; auto& mmContainer = getMMContainer(tid, pid, cid); bool lastTier = tid+1 >= getNumTiers(); mmContainer.withEvictionIterator([this, tid, pid, cid, &candidate, &toRecycle, &searchTries, &mmContainer, &lastTier, - &token](auto&& itr) { + &isExpired, &token](auto&& itr) { if (!itr) { ++searchTries; (*stats_.evictionAttempts)[tid][pid][cid].inc(); @@ -1634,7 +1635,7 @@ CacheAllocator::getNextCandidate(TierId tid, continue; } - auto marked = lastTier ? candidate_->markForEviction() : candidate_->markMoving(true); + auto marked = (lastTier || candidate_->isExpired()) ? candidate_->markForEviction() : candidate_->markMoving(true); if (!marked) { if (candidate_->hasChainedItem()) { stats_.evictFailParentAC.inc(); @@ -1651,6 +1652,7 @@ CacheAllocator::getNextCandidate(TierId tid, // since we won't be moving the item to the next tier toRecycle = toRecycle_; candidate = candidate_; + isExpired = candidate_->isExpired(); token = std::move(token_); // Check if parent changed for chained items - if yes, we cannot @@ -1673,7 +1675,7 @@ CacheAllocator::getNextCandidate(TierId tid, XDCHECK(candidate); XDCHECK(candidate->isMoving() || candidate->isMarkedForEviction()); - auto evictedToNext = lastTier ? nullptr + auto evictedToNext = (lastTier || isExpired) ? nullptr : tryEvictToNextMemoryTier(*candidate, false); if (!evictedToNext) { //if insertOrReplace was called during move @@ -1699,7 +1701,7 @@ CacheAllocator::getNextCandidate(TierId tid, // as exclusive since we will not be moving the item to the next tier // but rather just evicting all together, no need to // markForEvictionWhenMoving - auto ret = lastTier ? true : candidate->markForEvictionWhenMoving(); + auto ret = (lastTier || isExpired) ? true : candidate->markForEvictionWhenMoving(); XDCHECK(ret); unlinkItemForEviction(*candidate); @@ -1824,11 +1826,6 @@ CacheAllocator::tryEvictToNextMemoryTier( XDCHECK(item.isMoving()); XDCHECK(item.getRefCount() == 0); if(item.hasChainedItem()) return WriteHandle{}; // TODO: We do not support ChainedItem yet - if(item.isExpired()) { - accessContainer_->remove(item); - item.unmarkMoving(); - return acquire(&item); - } TierId nextTier = tid; // TODO - calculate this based on some admission policy while (++nextTier < getNumTiers()) { // try to evict down to the next memory tiers From 5632d1873b6d152f81e5764442dd592e9d63b075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Chor=C4=85=C5=BCewicz?= Date: Thu, 8 Jun 2023 12:24:04 -0700 Subject: [PATCH 25/31] Add option to insert items to first free tier (#87) instead of always inserting to topmost tier --- cachelib/allocator/CacheAllocator-inl.h | 27 ++++++++++++++++++----- cachelib/allocator/CacheAllocator.h | 8 ++++++- cachelib/allocator/CacheAllocatorConfig.h | 15 +++++++++++++ cachelib/cachebench/cache/Cache-inl.h | 2 ++ cachelib/cachebench/util/CacheConfig.cpp | 2 ++ cachelib/cachebench/util/CacheConfig.h | 2 ++ 6 files changed, 49 insertions(+), 7 deletions(-) diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 6a83559545..231b5631b4 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -424,7 +424,8 @@ CacheAllocator::allocateInternalTier(TierId tid, uint32_t size, uint32_t creationTime, uint32_t expiryTime, - bool fromBgThread) { + bool fromBgThread, + bool evict) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -445,7 +446,9 @@ CacheAllocator::allocateInternalTier(TierId tid, backgroundEvictor_[backgroundWorkerId(tid, pid, cid, backgroundEvictor_.size())]->wakeUp(); } - if (memory == nullptr) { + if (memory == nullptr && !evict) { + return {}; + } else if (memory == nullptr) { memory = findEviction(tid, pid, cid); } @@ -495,7 +498,8 @@ CacheAllocator::allocateInternal(PoolId pid, bool fromBgThread) { auto tid = 0; /* TODO: consult admission policy */ for(TierId tid = 0; tid < getNumTiers(); ++tid) { - auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime, fromBgThread); + bool evict = !config_.insertToFirstFreeTier || tid == getNumTiers() - 1; + auto handle = allocateInternalTier(tid, pid, key, size, creationTime, expiryTime, fromBgThread, evict); if (handle) return handle; } return {}; @@ -1829,13 +1833,17 @@ CacheAllocator::tryEvictToNextMemoryTier( TierId nextTier = tid; // TODO - calculate this based on some admission policy while (++nextTier < getNumTiers()) { // try to evict down to the next memory tiers + // always evict item from the nextTier to make room for new item + bool evict = true; + // allocateInternal might trigger another eviction auto newItemHdl = allocateInternalTier(nextTier, pid, item.getKey(), item.getSize(), item.getCreationTime(), item.getExpiryTime(), - fromBgThread); + fromBgThread, + evict); if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); @@ -1871,13 +1879,17 @@ CacheAllocator::tryPromoteToNextMemoryTier( auto toPromoteTier = nextTier - 1; --nextTier; + // always evict item from the toPromoteTier to make room for new item + bool evict = true; + // allocateInternal might trigger another eviction auto newItemHdl = allocateInternalTier(toPromoteTier, pid, item.getKey(), item.getSize(), item.getCreationTime(), item.getExpiryTime(), - fromBgThread); + fromBgThread, + true); if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); @@ -3251,6 +3263,8 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { const auto allocInfo = allocator_[getTierId(oldItem)]->getAllocInfo(static_cast(&oldItem)); + + bool evict = !config_.insertToFirstFreeTier || getTierId(oldItem) == getNumTiers() - 1; // Set up the destination for the move. Since oldItem would have the moving // bit set, it won't be picked for eviction. @@ -3260,7 +3274,8 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { oldItem.getSize(), oldItem.getCreationTime(), oldItem.getExpiryTime(), - false); + false, + evict); if (!newItemHdl) { return {}; } diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index e571da06e6..a3a528269d 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1521,13 +1521,19 @@ class CacheAllocator : public CacheBase { // For description see allocateInternal. // // @param tid id a memory tier + // @param fromBgThread whether this function was called from a bg + // thread - this is used to decide whether bg thread should + // be waken in case there is no free memory + // @param evict whether to evict an item from tier tid in case there + // is not enough memory WriteHandle allocateInternalTier(TierId tid, PoolId id, Key key, uint32_t size, uint32_t creationTime, uint32_t expiryTime, - bool fromBgThread); + bool fromBgThread, + bool evict); // Allocate a chained item // diff --git a/cachelib/allocator/CacheAllocatorConfig.h b/cachelib/allocator/CacheAllocatorConfig.h index 925b07706b..d4a9bd04a9 100644 --- a/cachelib/allocator/CacheAllocatorConfig.h +++ b/cachelib/allocator/CacheAllocatorConfig.h @@ -311,6 +311,9 @@ class CacheAllocatorConfig { // Library team if you find yourself customizing this. CacheAllocatorConfig& setThrottlerConfig(util::Throttler::Config config); + // Insert items to first free memory tier + CacheAllocatorConfig& enableInsertToFirstFreeTier(); + // Passes in a callback to initialize an event tracker when the allocator // starts CacheAllocatorConfig& setEventTracker(EventTrackerSharedPtr&&); @@ -527,6 +530,11 @@ class CacheAllocatorConfig { // ABOVE are the config for various cache workers // + // if turned off, always insert new elements to topmost memory tier. + // if turned on, insert new element to first free memory tier or evict memory + // from the bottom one if memory cache is full + bool insertToFirstFreeTier = false; + // the number of tries to search for an item to evict // 0 means it's infinite unsigned int evictionSearchTries{50}; @@ -662,6 +670,12 @@ class CacheAllocatorConfig { {MemoryTierCacheConfig::fromShm().setRatio(1)}}; }; +template +CacheAllocatorConfig& CacheAllocatorConfig::enableInsertToFirstFreeTier() { + insertToFirstFreeTier = true; + return *this; +} + template CacheAllocatorConfig& CacheAllocatorConfig::setCacheName( const std::string& _cacheName) { @@ -1241,6 +1255,7 @@ std::map CacheAllocatorConfig::serialize() const { configMap["nvmAdmissionMinTTL"] = std::to_string(nvmAdmissionMinTTL); configMap["delayCacheWorkersStart"] = delayCacheWorkersStart ? "true" : "false"; + configMap["insertToFirstFreeTier"] = std::to_string(insertToFirstFreeTier); mergeWithPrefix(configMap, throttleConfig.serialize(), "throttleConfig"); mergeWithPrefix(configMap, chainedItemAccessConfig.serialize(), diff --git a/cachelib/cachebench/cache/Cache-inl.h b/cachelib/cachebench/cache/Cache-inl.h index 36a69fe7a9..23481ad099 100644 --- a/cachelib/cachebench/cache/Cache-inl.h +++ b/cachelib/cachebench/cache/Cache-inl.h @@ -104,6 +104,8 @@ Cache::Cache(const CacheConfig& config, allocatorConfig_.configureMemoryTiers(config_.memoryTierConfigs); } + allocatorConfig_.insertToFirstFreeTier = config_.insertToFirstFreeTier; + auto cleanupGuard = folly::makeGuard([&] { if (!nvmCacheFilePath_.empty()) { util::removePath(nvmCacheFilePath_); diff --git a/cachelib/cachebench/util/CacheConfig.cpp b/cachelib/cachebench/util/CacheConfig.cpp index 5072021f9e..af5d7b4f64 100644 --- a/cachelib/cachebench/util/CacheConfig.cpp +++ b/cachelib/cachebench/util/CacheConfig.cpp @@ -49,6 +49,8 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) { JSONSetVal(configJson, tryLockUpdate); JSONSetVal(configJson, lruIpSpec); JSONSetVal(configJson, useCombinedLockForIterators); + + JSONSetVal(configJson, insertToFirstFreeTier); JSONSetVal(configJson, lru2qHotPct); JSONSetVal(configJson, lru2qColdPct); diff --git a/cachelib/cachebench/util/CacheConfig.h b/cachelib/cachebench/util/CacheConfig.h index 6481aa67c3..ec120c900a 100644 --- a/cachelib/cachebench/util/CacheConfig.h +++ b/cachelib/cachebench/util/CacheConfig.h @@ -97,6 +97,8 @@ struct CacheConfig : public JSONConfig { bool lruUpdateOnRead{true}; bool tryLockUpdate{false}; bool useCombinedLockForIterators{true}; + + bool insertToFirstFreeTier{false}; // LRU param uint64_t lruIpSpec{0}; From 09d7bab3ac06a561e683a099b120fa4494127acd Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Wed, 28 Jun 2023 13:12:32 -0400 Subject: [PATCH 26/31] Chained item movement between tiers - sync on the parent item (#84) * Chained item movement between tiers - currently sync on the parent item for moving. - updated tests accordingly, note that we can no longer swap parent item if chained item is being moved for slab release. * added some debug checks around chained item check * fix slab release behavior if no movecb --- cachelib/allocator/CacheAllocator-inl.h | 868 +++++++++--------- cachelib/allocator/CacheAllocator.h | 79 +- cachelib/allocator/CacheItem-inl.h | 4 +- cachelib/allocator/CacheItem.h | 6 +- cachelib/allocator/MM2Q-inl.h | 6 + cachelib/allocator/MM2Q.h | 4 + cachelib/allocator/MMLru-inl.h | 6 + cachelib/allocator/MMLru.h | 4 + cachelib/allocator/MMTinyLFU-inl.h | 7 + cachelib/allocator/MMTinyLFU.h | 4 + cachelib/allocator/Refcount.h | 16 +- .../allocator/tests/AllocatorTypeTest.cpp | 4 +- cachelib/allocator/tests/BaseAllocatorTest.h | 51 +- cachelib/allocator/tests/ItemTest.cpp | 2 +- .../allocator/tests/RebalanceStrategyTest.cpp | 3 + cachelib/allocator/tests/RefCountTest.cpp | 32 +- .../allocator/tests/SimpleRebalancingTest.h | 2 +- cachelib/cachebench/runner/CacheStressor.h | 6 +- .../test_configs/small_moving_bg.json | 35 + cachelib/common/Mutex.h | 6 + run_tests.sh | 1 + 21 files changed, 621 insertions(+), 525 deletions(-) create mode 100644 cachelib/cachebench/test_configs/small_moving_bg.json diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index 231b5631b4..a898d9b4fd 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -528,6 +528,18 @@ template typename CacheAllocator::WriteHandle CacheAllocator::allocateChainedItemInternal( const ReadHandle& parent, uint32_t size) { + auto tid = 0; /* TODO: consult admission policy */ + for(TierId tid = 0; tid < getNumTiers(); ++tid) { + auto handle = allocateChainedItemInternalTier(*parent, size, tid); + if (handle) return handle; + } + return {}; +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::allocateChainedItemInternalTier( + const Item& parent, uint32_t size, TierId tid) { util::LatencyTracker tracker{stats().allocateLatency_}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; @@ -535,11 +547,9 @@ CacheAllocator::allocateChainedItemInternal( // number of bytes required for this item const auto requiredSize = ChainedItem::getRequiredSize(size); - // TODO: is this correct? - auto tid = getTierId(*parent); - - const auto pid = allocator_[tid]->getAllocInfo(parent->getMemory()).poolId; - const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize); + const auto ptid = getTierId(parent); //it is okay because pools/classes are duplicated among the tiers + const auto pid = allocator_[ptid]->getAllocInfo(parent.getMemory()).poolId; + const auto cid = allocator_[ptid]->getAllocationClassId(pid, requiredSize); util::RollingLatencyTracker rollTracker{ (*stats_.classAllocLatency)[tid][pid][cid]}; @@ -558,7 +568,7 @@ CacheAllocator::allocateChainedItemInternal( SCOPE_FAIL { allocator_[tid]->free(memory); }; auto child = acquire( - new (memory) ChainedItem(compressor_.compress(parent.getInternal()), size, + new (memory) ChainedItem(compressor_.compress(&parent), size, util::getCurrentTimeSec())); if (child) { @@ -602,7 +612,7 @@ void CacheAllocator::addChainedItem(WriteHandle& parent, // Increment refcount since this chained item is now owned by the parent // Parent will decrement the refcount upon release. Since this is an // internal refcount, we dont include it in active handle tracking. - auto ret = child->incRef(true); + auto ret = child->incRef(); XDCHECK(ret == RefcountWithFlags::incResult::incOk); XDCHECK_EQ(2u, child->getRefCount()); @@ -671,22 +681,20 @@ CacheAllocator::getParentKey(const Item& chainedItem) { } template -void CacheAllocator::transferChainLocked(WriteHandle& parent, +void CacheAllocator::transferChainLocked(Item& parent, WriteHandle& newParent) { // parent must be in a state to not have concurrent readers. Eviction code - // paths rely on holding the last item handle. Since we hold on to an item - // handle here, the chain will not be touched by any eviction code path. - XDCHECK(parent); + // paths rely on holding the last item handle. XDCHECK(newParent); - XDCHECK_EQ(parent->getKey(), newParent->getKey()); - XDCHECK(parent->hasChainedItem()); + XDCHECK_EQ(parent.getKey(), newParent->getKey()); + XDCHECK(parent.hasChainedItem()); if (newParent->hasChainedItem()) { throw std::invalid_argument(folly::sformat( "New Parent {} has invalid state", newParent->toString())); } - auto headHandle = findChainedItem(*parent); + auto headHandle = findChainedItem(parent); XDCHECK(headHandle); // remove from the access container since we are changing the key @@ -698,6 +706,7 @@ void CacheAllocator::transferChainLocked(WriteHandle& parent, while (curr) { XDCHECK_EQ(curr == headHandle.get() ? 2u : 1u, curr->getRefCount()); XDCHECK(curr->isInMMContainer()); + XDCHECK(!newParent->isMoving()); curr->changeKey(newParentPtr); curr = curr->getNext(compressor_); } @@ -709,7 +718,7 @@ void CacheAllocator::transferChainLocked(WriteHandle& parent, folly::sformat("Did not expect to find an existing chain for {}", newParent->toString(), oldHead->toString())); } - parent->unmarkHasChainedItem(); + parent.unmarkHasChainedItem(); } template @@ -720,7 +729,7 @@ void CacheAllocator::transferChainAndReplace( } { // scope for chained item lock auto l = chainedItemLocks_.lockExclusive(parent->getKey()); - transferChainLocked(parent, newParent); + transferChainLocked(*parent, newParent); } if (replaceIfAccessible(*parent, *newParent)) { @@ -787,33 +796,10 @@ CacheAllocator::replaceChainedItem(Item& oldItem, } template -typename CacheAllocator::WriteHandle -CacheAllocator::replaceChainedItemLocked(Item& oldItem, - WriteHandle newItemHdl, - const Item& parent) { - XDCHECK(newItemHdl != nullptr); - XDCHECK_GE(1u, oldItem.getRefCount()); - - // grab the handle to the old item so that we can return this. Also, we need - // to drop the refcount the parent holds on oldItem by manually calling - // decRef. To do that safely we need to have a proper outstanding handle. - auto oldItemHdl = acquire(&oldItem); - - // Replace the old chained item with new item in the MMContainer before we - // actually replace the old item in the chain - - if (!replaceChainedItemInMMContainer(oldItem, *newItemHdl)) { - // This should never happen since we currently hold an valid - // parent handle. None of its chained items can be removed - throw std::runtime_error(folly::sformat( - "chained item cannot be replaced in MM container, oldItem={}, " - "newItem={}, parent={}", - oldItem.toString(), newItemHdl->toString(), parent.toString())); - } - - XDCHECK(!oldItem.isInMMContainer()); - XDCHECK(newItemHdl->isInMMContainer()); - +void CacheAllocator::replaceInChainLocked(Item& oldItem, + WriteHandle& newItemHdl, + const Item& parent, + bool fromMove) { auto head = findChainedItem(parent); XDCHECK(head != nullptr); XDCHECK_EQ(reinterpret_cast( @@ -842,17 +828,62 @@ CacheAllocator::replaceChainedItemLocked(Item& oldItem, oldItem.asChainedItem().getNext(compressor_), compressor_); oldItem.asChainedItem().setNext(nullptr, compressor_); - // this should not result in 0 refcount. We are bumping down the internal - // refcount. If it did, we would leak an item. - oldItem.decRef(); - XDCHECK_LT(0u, oldItem.getRefCount()) << oldItem.toString(); + //if called from moveChainedItem then ref will be zero, else + //greater than 0 + if (fromMove) { + //if this is the head chained item, release the handle now + //while refCount > 1 so that the destructor does not + //call releaseBackToAllocator since we want recycle oldItem + if (head) { + head.reset(); + XDCHECK_EQ(1u, oldItem.getRefCount()); + } + oldItem.decRef(); + XDCHECK_EQ(0u, oldItem.getRefCount()) << oldItem.toString(); + } else { + oldItem.decRef(); + XDCHECK_LT(0u, oldItem.getRefCount()) << oldItem.toString(); + } // increment refcount to indicate parent owns this similar to addChainedItem // Since this is an internal refcount, we dont include it in active handle // tracking. - auto ret = newItemHdl->incRef(true); + auto ret = newItemHdl->incRef(); XDCHECK(ret == RefcountWithFlags::incResult::incOk); +} + +template +typename CacheAllocator::WriteHandle +CacheAllocator::replaceChainedItemLocked(Item& oldItem, + WriteHandle newItemHdl, + const Item& parent) { + XDCHECK(newItemHdl != nullptr); + XDCHECK_GE(1u, oldItem.getRefCount()); + + // grab the handle to the old item so that we can return this. Also, we need + // to drop the refcount the parent holds on oldItem by manually calling + // decRef. To do that safely we need to have a proper outstanding handle. + auto oldItemHdl = acquire(&oldItem); + XDCHECK_GE(2u, oldItem.getRefCount()); + + // Replace the old chained item with new item in the MMContainer before we + // actually replace the old item in the chain + + if (!replaceChainedItemInMMContainer(oldItem, *newItemHdl)) { + // This should never happen since we currently hold an valid + // parent handle. None of its chained items can be removed + throw std::runtime_error(folly::sformat( + "chained item cannot be replaced in MM container, oldItem={}, " + "newItem={}, parent={}", + oldItem.toString(), newItemHdl->toString(), parent.toString())); + } + + XDCHECK(!oldItem.isInMMContainer()); + XDCHECK(newItemHdl->isInMMContainer()); + + replaceInChainLocked(oldItem, newItemHdl, parent, false); + return oldItemHdl; } @@ -885,7 +916,10 @@ CacheAllocator::releaseBackToAllocator(Item& it, // memory for a chained item but has decided not to insert the chained item // to a parent item and instead drop the chained item handle. In this case, // we free the chained item directly without calling remove callback. - if (it.isChainedItem()) { + // + // Except if we are moving a chained item between tiers - + // then it == toRecycle and we will want the normal recycle path + if (it.isChainedItem() && &it != toRecycle) { if (toRecycle) { throw std::runtime_error( folly::sformat("Can not recycle a chained item {}, toRecyle", @@ -958,10 +992,10 @@ CacheAllocator::releaseBackToAllocator(Item& it, while (head) { auto next = head->getNext(compressor_); - + const auto ctid = getTierId(head); const auto childInfo = - allocator_[tid]->getAllocInfo(static_cast(head)); - (*stats_.fragmentationSize)[tid][childInfo.poolId][childInfo.classId].sub( + allocator_[ctid]->getAllocInfo(static_cast(head)); + (*stats_.fragmentationSize)[ctid][childInfo.poolId][childInfo.classId].sub( util::getFragmentation(*this, *head)); removeFromMMContainer(*head); @@ -982,22 +1016,21 @@ CacheAllocator::releaseBackToAllocator(Item& it, // If the item is already moving and we already decremented the // refcount, we don't need to free this item. We'll let the slab // release thread take care of that - if (!wasMoving) { - if (childRef != 0) { - throw std::runtime_error(folly::sformat( - "chained item refcount is not zero. We cannot proceed! " - "Ref: {}, Chained Item: {}", - childRef, head->toString())); - } + XDCHECK(!wasMoving); + if (childRef != 0) { + throw std::runtime_error(folly::sformat( + "chained item refcount is not zero. We cannot proceed! " + "Ref: {}, Chained Item: {}", + childRef, head->toString())); + } - // Item is not moving and refcount is 0, we can proceed to - // free it or recylce the memory - if (head == toRecycle) { - XDCHECK(ReleaseRes::kReleased != res); - res = ReleaseRes::kRecycled; - } else { - allocator_[tid]->free(head); - } + // Item is not moving and refcount is 0, we can proceed to + // free it or recylce the memory + if (head == toRecycle) { + XDCHECK(ReleaseRes::kReleased != res); + res = ReleaseRes::kRecycled; + } else { + allocator_[ctid]->free(head); } stats_.numChainedChildItems.dec(); @@ -1007,6 +1040,7 @@ CacheAllocator::releaseBackToAllocator(Item& it, } if (&it == toRecycle) { + XDCHECK_EQ(it.getRefCount(),0u); XDCHECK(ReleaseRes::kReleased != res); res = ReleaseRes::kRecycled; } else { @@ -1018,8 +1052,8 @@ CacheAllocator::releaseBackToAllocator(Item& it, } template -RefcountWithFlags::incResult CacheAllocator::incRef(Item& it, bool failIfMoving) { - auto ret = it.incRef(failIfMoving); +RefcountWithFlags::incResult CacheAllocator::incRef(Item& it) { + auto ret = it.incRef(); if (ret == RefcountWithFlags::incResult::incOk) { ++handleCount_.tlStats(); } @@ -1043,11 +1077,8 @@ CacheAllocator::acquire(Item* it) { SCOPE_FAIL { stats_.numRefcountOverflow.inc(); }; - // TODO: do not block incRef for child items to avoid deadlock - const auto failIfMoving = getNumTiers() > 1 && !it->isChainedItem(); - while (true) { - auto incRes = incRef(*it, failIfMoving); + auto incRes = incRef(*it); if (LIKELY(incRes == RefcountWithFlags::incResult::incOk)) { return WriteHandle{it, *this}; } else if (incRes == RefcountWithFlags::incResult::incFailedEviction){ @@ -1219,7 +1250,6 @@ CacheAllocator::insertOrReplace(const WriteHandle& handle) { : std::unique_lock(); replaced = accessContainer_->insertOrReplace(*(handle.getInternal())); - if (replaced && replaced->isNvmClean() && !replaced->isNvmEvicted()) { // item is to be replaced and the destructor will be executed // upon memory released, mark it in nvm to avoid destructor @@ -1268,7 +1298,7 @@ CacheAllocator::insertOrReplace(const WriteHandle& handle) { /* Next two methods are used to asynchronously move Item between memory tiers. * * The thread, which moves Item, allocates new Item in the tier we are moving to - * and calls moveRegularItemWithSync() method. This method does the following: + * and calls moveRegularItem() method. This method does the following: * 1. Update the access container with the new item from the tier we are * moving to. This Item has moving flag set. * 2. Copy data from the old Item to the new one. @@ -1324,21 +1354,8 @@ size_t CacheAllocator::wakeUpWaitersLocked(folly::StringPiece key, } template -bool CacheAllocator::moveRegularItemWithSync( +bool CacheAllocator::moveRegularItem( Item& oldItem, WriteHandle& newItemHdl) { - //on function exit - the new item handle is no longer moving - //and other threads may access it - but in case where - //we failed to replace in access container we can give the - //new item back to the allocator - auto guard = folly::makeGuard([&]() { - auto ref = newItemHdl->unmarkMoving(); - if (UNLIKELY(ref == 0)) { - const auto res = - releaseBackToAllocator(*newItemHdl, RemoveContext::kNormal, false); - XDCHECK(res == ReleaseRes::kReleased); - } - }); - XDCHECK(oldItem.isMoving()); XDCHECK(!oldItem.isExpired()); // TODO: should we introduce new latency tracker. E.g. evictRegularLatency_ @@ -1353,38 +1370,6 @@ bool CacheAllocator::moveRegularItemWithSync( newItemHdl->markNvmClean(); } - // mark new item as moving to block readers until the data is copied - // (moveCb is called). Mark item in MMContainer temporarily (TODO: should - // we remove markMoving requirement for the item to be linked?) - newItemHdl->markInMMContainer(); - auto marked = newItemHdl->markMoving(false /* there is already a handle */); - newItemHdl->unmarkInMMContainer(); - XDCHECK(marked); - - auto predicate = [&](const Item& item){ - // we rely on moving flag being set (it should block all readers) - XDCHECK(item.getRefCount() == 0); - return true; - }; - - auto replaced = accessContainer_->replaceIf(oldItem, *newItemHdl, - predicate); - // another thread may have called insertOrReplace which could have - // marked this item as unaccessible causing the replaceIf - // in the access container to fail - in this case we want - // to abort the move since the item is no longer valid - if (!replaced) { - return false; - } - // what if another thread calls insertOrReplace now when - // the item is moving and already replaced in the hash table? - // 1. it succeeds in updating the hash table - so there is - // no guarentee that isAccessible() is true - // 2. it will then try to remove from MM container - // - this operation will wait for newItemHdl to - // be unmarkedMoving via the waitContext - // 3. replaced handle is returned and eventually drops - // ref to 0 and the item is recycled back to allocator. if (config_.moveCb) { // Execute the move callback. We cannot make any guarantees about the @@ -1405,17 +1390,24 @@ bool CacheAllocator::moveRegularItemWithSync( auto mmContainerAdded = newContainer.add(*newItemHdl); XDCHECK(mmContainerAdded); - // no one can add or remove chained items at this point + + auto predicate = [&](const Item& item){ + // we rely on moving flag being set (it should block all readers) + XDCHECK_EQ(item.getRefCount(),0); + XDCHECK(item.isMoving()); + return item.isMoving(); + }; + if (oldItem.hasChainedItem()) { - // safe to acquire handle for a moving Item - auto incRes = incRef(oldItem, false); - XDCHECK(incRes == RefcountWithFlags::incResult::incOk); - auto oldHandle = WriteHandle{&oldItem,*this}; - XDCHECK_EQ(1u, oldHandle->getRefCount()) << oldHandle->toString(); XDCHECK(!newItemHdl->hasChainedItem()) << newItemHdl->toString(); try { - auto l = chainedItemLocks_.lockExclusive(oldItem.getKey()); - transferChainLocked(oldHandle, newItemHdl); + auto l = chainedItemLocks_.tryLockExclusive(oldItem.getKey()); + if (l) { + transferChainLocked(oldItem, newItemHdl); + } else { + newContainer.remove(*newItemHdl); + return false; + } } catch (const std::exception& e) { // this should never happen because we drained all the handles. XLOGF(DFATAL, "{}", e.what()); @@ -1425,146 +1417,75 @@ bool CacheAllocator::moveRegularItemWithSync( XDCHECK(!oldItem.hasChainedItem()); XDCHECK(newItemHdl->hasChainedItem()); } - newItemHdl.unmarkNascent(); - return true; -} -template -bool CacheAllocator::moveRegularItem(Item& oldItem, - WriteHandle& newItemHdl) { - XDCHECK(config_.moveCb); - util::LatencyTracker tracker{stats_.moveRegularLatency_}; - - if (!oldItem.isAccessible() || oldItem.isExpired()) { + if (!accessContainer_->replaceIf(oldItem, *newItemHdl, predicate)) { + newContainer.remove(*newItemHdl); return false; } - XDCHECK_EQ(newItemHdl->getSize(), oldItem.getSize()); - XDCHECK_EQ(reinterpret_cast(&getMMContainer(oldItem)), - reinterpret_cast(&getMMContainer(*newItemHdl))); - - // take care of the flags before we expose the item to be accessed. this - // will ensure that when another thread removes the item from RAM, we issue - // a delete accordingly. See D7859775 for an example - if (oldItem.isNvmClean()) { - newItemHdl->markNvmClean(); - } - - // Execute the move callback. We cannot make any guarantees about the - // consistency of the old item beyond this point, because the callback can - // do more than a simple memcpy() e.g. update external references. If there - // are any remaining handles to the old item, it is the caller's - // responsibility to invalidate them. The move can only fail after this - // statement if the old item has been removed or replaced, in which case it - // should be fine for it to be left in an inconsistent state. - config_.moveCb(oldItem, *newItemHdl, nullptr); - - // Inside the access container's lock, this checks if the old item is - // accessible and its refcount is one. If the item is not accessible, - // there is no point to replace it since it had already been removed - // or in the process of being removed. If the item is in cache but the - // refcount is non-one, it means user could be attempting to remove - // this item through an API such as remove(itemHandle). In this case, - // it is unsafe to replace the old item with a new one, so we should - // also abort. - if (!accessContainer_->replaceIf(oldItem, *newItemHdl, - itemSlabMovePredicate)) { - return false; - } - - // Inside the MM container's lock, this checks if the old item exists to - // make sure that no other thread removed it, and only then replaces it. - if (!replaceInMMContainer(oldItem, *newItemHdl)) { - accessContainer_->remove(*newItemHdl); - return false; - } - - // Replacing into the MM container was successful, but someone could have - // called insertOrReplace() or remove() before or after the - // replaceInMMContainer() operation, which would invalidate newItemHdl. - if (!newItemHdl->isAccessible()) { - removeFromMMContainer(*newItemHdl); - return false; - } - - // no one can add or remove chained items at this point - if (oldItem.hasChainedItem()) { - auto oldItemHdl = acquire(&oldItem); - XDCHECK_EQ(1u, oldItemHdl->getRefCount()) << oldItemHdl->toString(); - XDCHECK(!newItemHdl->hasChainedItem()) << newItemHdl->toString(); - try { - auto l = chainedItemLocks_.lockExclusive(oldItem.getKey()); - transferChainLocked(oldItemHdl, newItemHdl); - } catch (const std::exception& e) { - // this should never happen because we drained all the handles. - XLOGF(DFATAL, "{}", e.what()); - throw; - } - - XDCHECK(!oldItem.hasChainedItem()); - XDCHECK(newItemHdl->hasChainedItem()); - } newItemHdl.unmarkNascent(); return true; } template bool CacheAllocator::moveChainedItem(ChainedItem& oldItem, - WriteHandle& newItemHdl) { - XDCHECK(config_.moveCb); + WriteHandle& newItemHdl, + Item& parentItem) { + XDCHECK(parentItem.isMoving()); util::LatencyTracker tracker{stats_.moveChainedLatency_}; - // This item has been unlinked from its parent and we're the only - // owner of it, so we're done here - if (!oldItem.isInMMContainer() || oldItem.isOnlyMoving()) { - return false; - } - auto& expectedParent = oldItem.getParentItem(compressor_); const auto parentKey = expectedParent.getKey(); - auto l = chainedItemLocks_.lockExclusive(parentKey); - - // verify old item under the lock - auto parentHandle = - validateAndGetParentHandleForChainedMoveLocked(oldItem, parentKey); - if (!parentHandle || &expectedParent != parentHandle.get()) { - return false; + auto l = chainedItemLocks_.tryLockExclusive(parentKey); + if (!l) { + return false; } + XDCHECK_EQ(&expectedParent,&parentItem); - // once we have the moving sync and valid parent for the old item, check if + // check if // the original allocation was made correctly. If not, we destroy the // allocation to indicate a retry to moving logic above. if (reinterpret_cast( &newItemHdl->asChainedItem().getParentItem(compressor_)) != - reinterpret_cast(&parentHandle->asChainedItem())) { - newItemHdl.reset(); + reinterpret_cast(&parentItem.asChainedItem())) { + XDCHECK(false); return false; } XDCHECK_EQ(reinterpret_cast( &newItemHdl->asChainedItem().getParentItem(compressor_)), - reinterpret_cast(&parentHandle->asChainedItem())); + reinterpret_cast(&parentItem.asChainedItem())); - // In case someone else had removed this chained item from its parent by now - // So we check again to see if the it has been unlinked from its parent - if (!oldItem.isInMMContainer() || oldItem.isOnlyMoving()) { - return false; - } - - auto parentPtr = parentHandle.getInternal(); + auto parentPtr = &parentItem; XDCHECK_EQ(reinterpret_cast(parentPtr), reinterpret_cast(&oldItem.getParentItem(compressor_))); - // Invoke the move callback to fix up any user data related to the chain - config_.moveCb(oldItem, *newItemHdl, parentPtr); + if (config_.moveCb) { + // Execute the move callback. We cannot make any guarantees about the + // consistency of the old item beyond this point, because the callback can + // do more than a simple memcpy() e.g. update external references. If there + // are any remaining handles to the old item, it is the caller's + // responsibility to invalidate them. The move can only fail after this + // statement if the old item has been removed or replaced, in which case it + // should be fine for it to be left in an inconsistent state. + config_.moveCb(oldItem, *newItemHdl, parentPtr); + } else { + std::memcpy(newItemHdl->getMemory(), oldItem.getMemory(), + oldItem.getSize()); + } // Replace the new item in the position of the old one before both in the // parent's chain and the MMContainer. - auto oldItemHandle = - replaceChainedItemLocked(oldItem, std::move(newItemHdl), *parentHandle); - XDCHECK(oldItemHandle->isMoving()); - XDCHECK(!oldItemHandle->isInMMContainer()); + XDCHECK_EQ(parentItem.getRefCount(),0); + XDCHECK(parentItem.isMoving()); + XDCHECK(l); + + auto& newContainer = getMMContainer(*newItemHdl); + auto mmContainerAdded = newContainer.add(*newItemHdl); + XDCHECK(mmContainerAdded); + + replaceInChainLocked(oldItem, newItemHdl, parentItem, true); return true; } @@ -1599,12 +1520,17 @@ CacheAllocator::getNextCandidate(TierId tid, unsigned int& searchTries) { typename NvmCacheT::PutToken token; Item* toRecycle = nullptr; + Item* toRecycleParent = nullptr; Item* candidate = nullptr; bool isExpired = false; + Item* syncItem = nullptr; + bool chainedItem = false; auto& mmContainer = getMMContainer(tid, pid, cid); bool lastTier = tid+1 >= getNumTiers(); - mmContainer.withEvictionIterator([this, tid, pid, cid, &candidate, &toRecycle, + mmContainer.withEvictionIterator([this, tid, pid, cid, &candidate, + &toRecycle, &toRecycleParent, &syncItem, + &chainedItem, &searchTries, &mmContainer, &lastTier, &isExpired, &token](auto&& itr) { if (!itr) { @@ -1620,11 +1546,36 @@ CacheAllocator::getNextCandidate(TierId tid, (*stats_.evictionAttempts)[tid][pid][cid].inc(); auto* toRecycle_ = itr.get(); - auto* candidate_ = - toRecycle_->isChainedItem() + bool chainedItem_ = toRecycle_->isChainedItem(); + Item* toRecycleParent_ = chainedItem_ ? &toRecycle_->asChainedItem().getParentItem(compressor_) - : toRecycle_; - + : nullptr; + // in order to safely check if the expected parent (toRecycleParent_) matches + // the current parent on the chained item, we need to take the chained + // item lock so we are sure that nobody else will be editing the chain + auto l_ = chainedItem_ + ? chainedItemLocks_.tryLockExclusive(toRecycleParent_->getKey()) + : decltype(chainedItemLocks_.tryLockExclusive(toRecycle_->getKey()))(); + + if (chainedItem_ && + ( !l_ || &toRecycle_->asChainedItem().getParentItem(compressor_) + != toRecycleParent_) ) { + ++itr; + continue; + } + Item* candidate_; + Item* syncItem_; + //sync on the parent item for chained items to move to next tier + if (!lastTier && chainedItem_) { + syncItem_ = toRecycleParent_; + candidate_ = toRecycle_; + } else if (lastTier && chainedItem_) { + candidate_ = toRecycleParent_; + syncItem_ = toRecycleParent_; + } else { + candidate_ = toRecycle_; + syncItem_ = toRecycle_; + } // if it's last tier, the item will be evicted // need to create put token before marking it exclusive const bool evictToNvmCache = lastTier && shouldWriteToNvmCache(*candidate_); @@ -1632,14 +1583,14 @@ CacheAllocator::getNextCandidate(TierId tid, auto token_ = evictToNvmCache ? nvmCache_->createPutToken(candidate_->getKey()) : typename NvmCacheT::PutToken{}; - + if (evictToNvmCache && !token_.isValid()) { stats_.evictFailConcurrentFill.inc(); ++itr; continue; } - - auto marked = (lastTier || candidate_->isExpired()) ? candidate_->markForEviction() : candidate_->markMoving(true); + + auto marked = (lastTier || candidate_->isExpired()) ? syncItem_->markForEviction() : syncItem_->markMoving(); if (!marked) { if (candidate_->hasChainedItem()) { stats_.evictFailParentAC.inc(); @@ -1647,10 +1598,14 @@ CacheAllocator::getNextCandidate(TierId tid, stats_.evictFailAC.inc(); } ++itr; + XDCHECK_EQ(toRecycle,nullptr); + XDCHECK_EQ(candidate,nullptr); continue; } - - XDCHECK(candidate_->isMoving() || candidate_->isMarkedForEviction()); + + XDCHECK(syncItem_->isMoving() || syncItem_->isMarkedForEviction()); + toRecycleParent = toRecycleParent_; + chainedItem = chainedItem_; // markForEviction to make sure no other thead is evicting the item // nor holding a handle to that item if this is last tier // since we won't be moving the item to the next tier @@ -1658,15 +1613,12 @@ CacheAllocator::getNextCandidate(TierId tid, candidate = candidate_; isExpired = candidate_->isExpired(); token = std::move(token_); - - // Check if parent changed for chained items - if yes, we cannot - // remove the child from the mmContainer as we will not be evicting - // it. We could abort right here, but we need to cleanup in case - // unmarkForEviction() returns 0 - so just go through normal path. - if (!toRecycle_->isChainedItem() || - &toRecycle->asChainedItem().getParentItem(compressor_) == candidate) { - mmContainer.remove(itr); + if (chainedItem) { + XDCHECK(l_); + XDCHECK_EQ(toRecycleParent,&toRecycle_->asChainedItem().getParentItem(compressor_)); } + mmContainer.remove(itr); + return; } }); @@ -1682,6 +1634,14 @@ CacheAllocator::getNextCandidate(TierId tid, auto evictedToNext = (lastTier || isExpired) ? nullptr : tryEvictToNextMemoryTier(*candidate, false); if (!evictedToNext) { + //failed to move a chained item - so evict the entire chain + if (candidate->isChainedItem()) { + //candidate should be parent now + XDCHECK(toRecycleParent->isMoving()); + XDCHECK_EQ(candidate,toRecycle); + candidate = toRecycleParent; //but now we evict the chain and in + //doing so recycle the child + } //if insertOrReplace was called during move //then candidate will not be accessible (failed replace during tryEvict) // - therefore this was why we failed to @@ -1718,7 +1678,6 @@ CacheAllocator::getNextCandidate(TierId tid, // it's safe to do now, as we have the item marked exclusive and // no other reader can be added to the waiters list wakeUpWaiters(*candidate, {}); - } else { XDCHECK(!evictedToNext->isMarkedForEviction() && !evictedToNext->isMoving()); XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); @@ -1726,7 +1685,34 @@ CacheAllocator::getNextCandidate(TierId tid, XDCHECK(candidate->getKey() == evictedToNext->getKey()); (*stats_.numWritebacks)[tid][pid][cid].inc(); - wakeUpWaiters(*candidate, std::move(evictedToNext)); + if (chainedItem) { + XDCHECK(toRecycleParent->isMoving()); + XDCHECK_EQ(evictedToNext->getRefCount(),2u); + (*stats_.chainedItemEvictions)[tid][pid][cid].inc(); + // check if by releasing the item we intend to, we actually + // recycle the candidate. + auto ret = releaseBackToAllocator(*candidate, RemoveContext::kEviction, + /* isNascent */ false, toRecycle); + XDCHECK_EQ(ret,ReleaseRes::kRecycled); + evictedToNext.reset(); //once we unmark moving threads will try and alloc, drop + //the handle now - and refcount will drop to 1 + auto ref = toRecycleParent->unmarkMoving(); + if (UNLIKELY(ref == 0)) { + wakeUpWaiters(*toRecycleParent,{}); + const auto res = + releaseBackToAllocator(*toRecycleParent, RemoveContext::kNormal, false); + XDCHECK(res == ReleaseRes::kReleased); + } else { + auto parentHandle = acquire(toRecycleParent); + if (parentHandle) { + wakeUpWaiters(*toRecycleParent,std::move(parentHandle)); + } //in case where parent handle is null that means some other thread + // would have called wakeUpWaiters with null handle and released + // parent back to allocator + } + } else { + wakeUpWaiters(*candidate, std::move(evictedToNext)); + } } XDCHECK(!candidate->isMarkedForEviction() && !candidate->isMoving()); @@ -1764,6 +1750,7 @@ CacheAllocator::findEviction(TierId tid, PoolId pid, ClassId cid) { candidate->getConfiguredTTL().count()); } + XDCHECK(!candidate->isChainedItem()); // check if by releasing the item we intend to, we actually // recycle the candidate. auto ret = releaseBackToAllocator(*candidate, RemoveContext::kEviction, @@ -1827,9 +1814,6 @@ template typename CacheAllocator::WriteHandle CacheAllocator::tryEvictToNextMemoryTier( TierId tid, PoolId pid, Item& item, bool fromBgThread) { - XDCHECK(item.isMoving()); - XDCHECK(item.getRefCount() == 0); - if(item.hasChainedItem()) return WriteHandle{}; // TODO: We do not support ChainedItem yet TierId nextTier = tid; // TODO - calculate this based on some admission policy while (++nextTier < getNumTiers()) { // try to evict down to the next memory tiers @@ -1837,21 +1821,44 @@ CacheAllocator::tryEvictToNextMemoryTier( bool evict = true; // allocateInternal might trigger another eviction - auto newItemHdl = allocateInternalTier(nextTier, pid, + WriteHandle newItemHdl{}; + Item* parentItem; + bool chainedItem = false; + if(item.isChainedItem()) { + chainedItem = true; + parentItem = &item.asChainedItem().getParentItem(compressor_); + XDCHECK(parentItem->isMoving()); + XDCHECK(item.isChainedItem() && item.getRefCount() == 1); + XDCHECK_EQ(0, parentItem->getRefCount()); + newItemHdl = allocateChainedItemInternalTier(*parentItem, + item.getSize(), + nextTier); + } else { + // this assert can fail if parent changed + XDCHECK(item.isMoving()); + XDCHECK(item.getRefCount() == 0); + newItemHdl = allocateInternalTier(nextTier, pid, item.getKey(), item.getSize(), item.getCreationTime(), item.getExpiryTime(), fromBgThread, evict); + } if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); - if (!moveRegularItemWithSync(item, newItemHdl)) { - return WriteHandle{}; + bool moveSuccess = chainedItem + ? moveChainedItem(item.asChainedItem(), + newItemHdl, *parentItem) + : moveRegularItem(item, newItemHdl); + if (!moveSuccess) { + return WriteHandle{}; + } + if (!chainedItem) { + XDCHECK_EQ(newItemHdl->getKey(),item.getKey()); + item.unmarkMoving(); } - XDCHECK_EQ(newItemHdl->getKey(),item.getKey()); - item.unmarkMoving(); return newItemHdl; } else { return WriteHandle{}; @@ -1893,7 +1900,7 @@ CacheAllocator::tryPromoteToNextMemoryTier( if (newItemHdl) { XDCHECK_EQ(newItemHdl->getSize(), item.getSize()); - if (!moveRegularItemWithSync(item, newItemHdl)) { + if (!moveRegularItem(item, newItemHdl)) { return WriteHandle{}; } item.unmarkMoving(); @@ -3089,6 +3096,8 @@ void CacheAllocator::releaseSlabImpl(TierId tid, // 3. If 2 is successful, Move or Evict // 4. Move on to the next item if current item is freed for (auto alloc : releaseContext.getActiveAllocations()) { + Item& item = *static_cast(alloc); + // Need to mark an item for release before proceeding // If we can't mark as moving, it means the item is already freed const bool isAlreadyFreed = @@ -3097,8 +3106,6 @@ void CacheAllocator::releaseSlabImpl(TierId tid, continue; } - Item& item = *static_cast(alloc); - // Try to move this item and make sure we can free the memory const bool isMoved = moveForSlabRelease(releaseContext, item, throttler); @@ -3131,21 +3138,25 @@ typename RefcountWithFlags::Value CacheAllocator::unmarkMovingAndWak template bool CacheAllocator::moveForSlabRelease( const SlabReleaseContext& ctx, Item& oldItem, util::Throttler& throttler) { + if (!config_.moveCb) { - return false; + return false; } - bool isMoved = false; auto startTime = util::getCurrentTimeSec(); WriteHandle newItemHdl{}; + Item *parentItem; + bool chainedItem = oldItem.isChainedItem(); for (unsigned int itemMovingAttempts = 0; itemMovingAttempts < config_.movingTries; ++itemMovingAttempts) { stats_.numMoveAttempts.inc(); - // Nothing to move and the key is likely also bogus for chained items. + // Nothing to move - in the case that tryMoving failed + // for chained items we would have already evicted the entire chain. if (oldItem.isOnlyMoving()) { + XDCHECK(!oldItem.isChainedItem()); auto ret = unmarkMovingAndWakeUpWaiters(oldItem, {}); XDCHECK(ret == 0); const auto res = @@ -3163,7 +3174,17 @@ bool CacheAllocator::moveForSlabRelease( }); if (!newItemHdl) { - newItemHdl = allocateNewItemForOldItem(oldItem); + if (chainedItem) { + parentItem = &oldItem.asChainedItem().getParentItem(compressor_); + XDCHECK(parentItem->isMoving()); + XDCHECK(oldItem.isChainedItem() && oldItem.getRefCount() == 1); + XDCHECK_EQ(0, parentItem->getRefCount()); + newItemHdl = + allocateChainedItemInternalTier(*parentItem, oldItem.getSize(), getTierId(oldItem)); + } else { + XDCHECK(oldItem.isMoving()); + newItemHdl = allocateNewItemForOldItem(oldItem); + } } // if we have a valid handle, try to move, if not, we retry. @@ -3185,7 +3206,7 @@ bool CacheAllocator::moveForSlabRelease( // that's identical to this one to replace it. Here we just need to wait // until all users have dropped the item handles before we can proceed. startTime = util::getCurrentTimeSec(); - while (!oldItem.isOnlyMoving()) { + while (!chainedItem && !oldItem.isOnlyMoving()) { throttleWith(throttler, [&] { XLOGF(WARN, "Spent {} seconds, slab release still waiting for refcount to " @@ -3195,16 +3216,33 @@ bool CacheAllocator::moveForSlabRelease( }); } auto tid = getTierId(oldItem); - auto ref = unmarkMovingAndWakeUpWaiters(oldItem, std::move(newItemHdl)); - XDCHECK(ref == 0); - const auto allocInfo = allocator_[tid]->getAllocInfo(oldItem.getMemory()); + if (chainedItem) { + newItemHdl.reset(); + auto ref = parentItem->unmarkMoving(); + if (UNLIKELY(ref == 0)) { + wakeUpWaiters(*parentItem,{}); + const auto res = + releaseBackToAllocator(*parentItem, RemoveContext::kNormal, false); + XDCHECK(res == ReleaseRes::kReleased); + return true; + } else { + XDCHECK_NE(ref,0); + auto parentHdl = acquire(parentItem); + if (parentHdl) { + wakeUpWaiters(*parentItem,std::move(parentHdl)); + } + } + } else { + auto ref = unmarkMovingAndWakeUpWaiters(oldItem, std::move(newItemHdl)); + XDCHECK(ref == 0); + } allocator_[tid]->free(&oldItem); - (*stats_.fragmentationSize)[tid][allocInfo.poolId][allocInfo.classId].sub( util::getFragmentation(*this, oldItem)); stats_.numMoveSuccesses.inc(); return true; + } template @@ -3230,7 +3268,6 @@ CacheAllocator::validateAndGetParentHandleForChainedMoveLocked( template typename CacheAllocator::WriteHandle CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { - XDCHECK(oldItem.isMoving()); if (oldItem.isChainedItem()) { const auto& oldChainedItem = oldItem.asChainedItem(); const auto parentKey = oldChainedItem.getParentItem(compressor_).getKey(); @@ -3290,75 +3327,38 @@ CacheAllocator::allocateNewItemForOldItem(const Item& oldItem) { template bool CacheAllocator::tryMovingForSlabRelease( Item& oldItem, WriteHandle& newItemHdl) { - // By holding onto a user-level synchronization object, we ensure moving - // a regular item or chained item is synchronized with any potential - // user-side mutation. - std::unique_ptr syncObj; - if (config_.movingSync && getNumTiers() == 1) { - // TODO: use moving-bit synchronization for single tier as well - if (!oldItem.isChainedItem()) { - syncObj = config_.movingSync(oldItem.getKey()); - } else { - // Copy the key so we have a valid key to work with if the chained - // item is still valid. - const std::string parentKey = - oldItem.asChainedItem().getParentItem(compressor_).getKey().str(); - if (oldItem.isOnlyMoving()) { - // If chained item no longer has a refcount, its parent is already - // being released, so we abort this try to moving. - return false; + std::unique_ptr syncObj; + if (config_.movingSync) { + if (!oldItem.isChainedItem()) { + syncObj = config_.movingSync(oldItem.getKey()); + } else { + // Copy the key so we have a valid key to work with if the chained + // item is still valid. + const std::string parentKey = + oldItem.asChainedItem().getParentItem(compressor_).getKey().str(); + syncObj = config_.movingSync(parentKey); + } + if (syncObj && !syncObj->isValid()) { + return false; } - syncObj = config_.movingSync(parentKey); - } - - // We need to differentiate between the following three scenarios: - // 1. nullptr indicates no move sync required for this particular item - // 2. moveSync.isValid() == true meaning we've obtained the sync - // 3. moveSync.isValid() == false meaning we need to abort and retry - if (syncObj && !syncObj->isValid()) { - return false; - } - } - // TODO: we can unify move*Item and move*ItemWithSync by always - // using the moving bit to block readers. - if (getNumTiers() == 1) { - return oldItem.isChainedItem() - ? moveChainedItem(oldItem.asChainedItem(), newItemHdl) + } + //move can fail if another thread calls insertOrReplace + //in this case oldItem is no longer valid (not accessible, + //it gets removed from MMContainer and evictForSlabRelease + //will send it back to the allocator + bool ret = oldItem.isChainedItem() + ? moveChainedItem(oldItem.asChainedItem(), newItemHdl, + oldItem.asChainedItem().getParentItem(compressor_)) : moveRegularItem(oldItem, newItemHdl); - } else { - if (oldItem.isChainedItem() || oldItem.hasChainedItem()) { - // TODO: add support for chained items - return false; - } else { - //move can fail if another thread calls insertOrReplace - //in this case oldItem is no longer valid (not accessible, - //it gets removed from MMContainer and evictForSlabRelease - //will send it back to the allocator - bool ret = moveRegularItemWithSync(oldItem, newItemHdl); - if (!ret) { - //we failed to move - newItemHdl was released back to allocator - //by the moveRegularItemWithSync but oldItem is not accessible - //and no longer valid - we need to clean it up here - XDCHECK(!oldItem.isAccessible()); - oldItem.markForEvictionWhenMoving(); - unlinkItemForEviction(oldItem); - wakeUpWaiters(oldItem, {}); - } else { - removeFromMMContainer(oldItem); - } - return ret; - } - } + removeFromMMContainer(oldItem); + return ret; } template void CacheAllocator::wakeUpWaiters(Item& item, WriteHandle handle) { - // readers do not block on 'moving' items in case there is only one tier - if (getNumTiers() > 1) { - wakeUpWaitersLocked(item.getKey(), std::move(handle)); - } + wakeUpWaitersLocked(item.getKey(), std::move(handle)); } template @@ -3367,16 +3367,19 @@ void CacheAllocator::evictForSlabRelease( auto startTime = util::getCurrentTimeSec(); while (true) { - XDCHECK(item.isMoving()); + //we can't rely on an item being marked moving because + //it may have previously been a chained item stats_.numEvictionAttempts.inc(); if (shutDownInProgress_) { - auto ref = unmarkMovingAndWakeUpWaiters(item, {}); - allocator_[getTierId(item)]->abortSlabRelease(ctx); - throw exception::SlabReleaseAborted( - folly::sformat("Slab Release aborted while trying to evict" - " Item: {} Pool: {}, Class: {}.", - item.toString(), ctx.getPoolId(), ctx.getClassId())); + if (item.isMoving()) { + auto ref = unmarkMovingAndWakeUpWaiters(item, {}); + allocator_[getTierId(item)]->abortSlabRelease(ctx); + throw exception::SlabReleaseAborted( + folly::sformat("Slab Release aborted while trying to evict" + " Item: {} Pool: {}, Class: {}.", + item.toString(), ctx.getPoolId(), ctx.getClassId())); + } } throttleWith(throttler, [&] { XLOGF(WARN, @@ -3404,80 +3407,37 @@ void CacheAllocator::evictForSlabRelease( } typename NvmCacheT::PutToken token; + bool isChainedItem = item.isChainedItem(); Item* evicted; - if (item.isChainedItem()) { - auto& expectedParent = item.asChainedItem().getParentItem(compressor_); - - if (getNumTiers() == 1) { - // TODO: unify this with multi-tier implementation - // right now, taking a chained item lock here would lead to deadlock - const std::string parentKey = expectedParent.getKey().str(); - auto l = chainedItemLocks_.lockExclusive(parentKey); - - // check if the child is still in mmContainer and the expected parent is - // valid under the chained item lock. - if (expectedParent.getKey() != parentKey || !item.isInMMContainer() || - item.isOnlyMoving() || - &expectedParent != &item.asChainedItem().getParentItem(compressor_) || - !expectedParent.isAccessible() || !expectedParent.hasChainedItem()) { - continue; - } - - // search if the child is present in the chain - { - auto parentHandle = findInternal(parentKey); - if (!parentHandle || parentHandle != &expectedParent) { - continue; - } - - ChainedItem* head = nullptr; - { // scope for the handle - auto headHandle = findChainedItem(expectedParent); - head = headHandle ? &headHandle->asChainedItem() : nullptr; - } - - bool found = false; - while (head) { - if (head == &item) { - found = true; - break; - } - head = head->getNext(compressor_); - } - - if (!found) { - continue; - } - } - } - - evicted = &expectedParent; - token = createPutToken(*evicted); - if (evicted->markForEviction()) { - // unmark the child so it will be freed - item.unmarkMoving(); - unlinkItemForEviction(*evicted); - // wake up any readers that wait for the move to complete - // it's safe to do now, as we have the item marked exclusive and - // no other reader can be added to the waiters list - wakeUpWaiters(*evicted, {}); - } else { - // TODO: potential deadlock with markUseful for parent item - // for now, we do not block any reader on child items but this - // should probably be fixed - continue; + Item *expectedParent = isChainedItem + ? &item.asChainedItem().getParentItem(compressor_) + : nullptr; + if (isChainedItem) { + XDCHECK(expectedParent->isMoving()); + XDCHECK_EQ(expectedParent,&item.asChainedItem().getParentItem(compressor_)); + if (expectedParent != &item.asChainedItem().getParentItem(compressor_)) { + XDCHECK_EQ(expectedParent,&item.asChainedItem().getParentItem(compressor_)); + throw std::runtime_error(folly::sformat( + "Slab release aborted while evicting " + "item {}", item.toString())); } + evicted = expectedParent; } else { evicted = &item; - - token = createPutToken(*evicted); - if (evicted->markForEvictionWhenMoving()) { - unlinkItemForEviction(*evicted); - wakeUpWaiters(*evicted, {}); - } else { - continue; - } } + XDCHECK(evicted->isMoving()); + token = createPutToken(*evicted); + auto ret = evicted->markForEvictionWhenMoving(); + XDCHECK(ret); + // unmark the child so it will be freed + // TODO entire chain just gets evicted since moveForSlabRelease + // returns false + XDCHECK(!item.isMoving()); + unlinkItemForEviction(*evicted); + // wake up any readers that wait for the move to complete + // it's safe to do now, as we have the item marked exclusive and + // no other reader can be added to the waiters list + wakeUpWaiters(*evicted, {}); if (token.isValid() && shouldWriteToNvmCacheExclusive(*evicted)) { nvmCache_->put(*evicted, std::move(token)); @@ -3498,14 +3458,10 @@ void CacheAllocator::evictForSlabRelease( const auto res = releaseBackToAllocator(*evicted, RemoveContext::kEviction, false); - if (getNumTiers() == 1) { - XDCHECK(res == ReleaseRes::kReleased); - } else { - const bool isAlreadyFreed = - !markMovingForSlabRelease(ctx, &item, throttler); - if (!isAlreadyFreed) { - continue; - } + const bool isAlreadyFreed = + !markMovingForSlabRelease(ctx, &item, throttler); + if (!isAlreadyFreed) { + continue; } return; @@ -3553,19 +3509,52 @@ bool CacheAllocator::markMovingForSlabRelease( // At first, we assume this item was already freed bool itemFreed = true; + Item *syncItem = nullptr; bool markedMoving = false; TierId tid = getTierId(alloc); - auto numTiers = getNumTiers(); - const auto fn = [&markedMoving, &itemFreed, numTiers](void* memory) { + const auto fn = [this, tid, &syncItem, &markedMoving, &itemFreed](void* memory) { // Since this callback is executed, the item is not yet freed itemFreed = false; Item* item = static_cast(memory); - // TODO: for chained items, moving bit is only used to avoid - // freeing the item prematurely - auto failIfRefNotZero = numTiers > 1 && !item->isChainedItem(); - if (item->markMoving(failIfRefNotZero)) { - markedMoving = true; - } + auto allocInfo = allocator_[tid]->getAllocInfo(memory); + auto pid = allocInfo.poolId; + auto cid = allocInfo.classId; + auto& mmContainer = getMMContainer(tid, pid, cid); + mmContainer.withContainerLock([this, &mmContainer, + &syncItem, &item, &markedMoving]() { + //we rely on the mmContainer lock to safely check that the item is + //currently in the mmContainer (no other threads are currently allocating + //this item). This is needed to sync on the case where a chained item + //is being released back to allocator and it's parent ref could be + //invalid. We need a valid parent ref in order to mark a chained item + //as moving since we sync on the parent by marking it as moving. + if (!item->isInMMContainer()) { + return; + } + bool chainedItem_ = item->isChainedItem(); + XDCHECK_EQ(&getMMContainer(*item),&mmContainer); + XDCHECK_EQ(item->isChainedItem(),chainedItem_); + Item* syncItem_ = chainedItem_ + ? &item->asChainedItem().getParentItem(compressor_) + : item; + // in order to safely check if the expected parent (syncItem_) matches + // the current parent on the chained item, we need to take the chained + // item lock so we are sure that nobody else will be editing the chain + auto l_ = chainedItem_ + ? chainedItemLocks_.tryLockExclusive(syncItem_->getKey()) + : decltype(chainedItemLocks_.tryLockExclusive(syncItem_->getKey()))(); + + XDCHECK_EQ(item->isChainedItem(),chainedItem_); + if (chainedItem_ && + ( !l_ || &item->asChainedItem().getParentItem(compressor_) != syncItem_) ) { + markedMoving = false; + return; + } + if (syncItem_->markMoving()) { + markedMoving = true; + syncItem = syncItem_; + } + }); }; auto startTime = util::getCurrentTimeSec(); @@ -3577,6 +3566,24 @@ bool CacheAllocator::markMovingForSlabRelease( if (itemFreed) { return false; } else if (markedMoving) { + Item* item = static_cast(alloc); + XDCHECK(syncItem->isMoving()); + XDCHECK(item->isChainedItem() + ? item->asChainedItem().getParentItem(compressor_).isMoving() + : item->isMoving()) << item->toString() << "\n" << syncItem->toString(); + if ( ( item->isChainedItem() && + !item->asChainedItem().getParentItem(compressor_).isMoving() ) + || (!item->isChainedItem() && !item->isMoving()) ) { + throw std::runtime_error( + folly::sformat("Slab Release aborted - failed to mark" + " as moving for Item: {}. Pool: {}, Class: {}. Parent is {}", + item->toString(), ctx.getPoolId(), + ctx.getClassId(), + item->isChainedItem() + ? item->asChainedItem().getParentItem(compressor_).toString() + : "none")); + + } return true; } @@ -3592,6 +3599,7 @@ bool CacheAllocator::markMovingForSlabRelease( static_cast(alloc)->toString(), ctx.getPoolId(), ctx.getClassId())); } + stats_.numMoveAttempts.inc(); throttleWith(throttler, [&] { XLOGF(WARN, "Spent {} seconds, slab release still trying to mark as moving for " diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index a3a528269d..0ec7224603 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -1394,7 +1394,7 @@ class CacheAllocator : public CacheBase { private: // wrapper around Item's refcount and active handle tracking - FOLLY_ALWAYS_INLINE RefcountWithFlags::incResult incRef(Item& it, bool failIfMoving); + FOLLY_ALWAYS_INLINE RefcountWithFlags::incResult incRef(Item& it); FOLLY_ALWAYS_INLINE RefcountWithFlags::Value decRef(Item& it); // drops the refcount and if needed, frees the allocation back to the memory @@ -1552,6 +1552,26 @@ class CacheAllocator : public CacheBase { WriteHandle allocateChainedItemInternal(const ReadHandle& parent, uint32_t size); + // Allocate a chained item to a specific tier + // + // The resulting chained item does not have a parent item yet + // and if we fail to link to the chain for any reasoin + // the chained item will be freed once the handle is dropped. + // + // The parent item parameter here is mainly used to find the + // correct pool to allocate memory for this chained item + // + // @param parent parent item + // @param size the size for the chained allocation + // @param tid the tier to allocate on + // + // @return handle to the chained allocation + // @throw std::invalid_argument if the size requested is invalid or + // if the item is invalid + WriteHandle allocateChainedItemInternalTier(const Item& parent, + uint32_t size, + TierId tid); + // Given an item and its parentKey, validate that the parentKey // corresponds to an item that's the parent of the supplied item. // @@ -1632,19 +1652,17 @@ class CacheAllocator : public CacheBase { // // @return true If the move was completed, and the containers were updated // successfully. - bool moveRegularItemWithSync(Item& oldItem, WriteHandle& newItemHdl); + bool moveRegularItem(Item& oldItem, WriteHandle& newItemHdl); - // Moves a regular item to a different slab. This should only be used during - // slab release after the item's exclusive bit has been set. The user supplied - // callback is responsible for copying the contents and fixing the semantics - // of chained item. + // Moves a chained item to a different memory tier. // - // @param oldItem item being moved + // @param oldItem Reference to the item being moved // @param newItemHdl Reference to the handle of the new item being moved into + // @param parentHandle Reference to the handle of the parent item // // @return true If the move was completed, and the containers were updated // successfully. - bool moveRegularItem(Item& oldItem, WriteHandle& newItemHdl); + bool moveChainedItem(ChainedItem& oldItem, WriteHandle& newItemHdl, Item& parentItem); // template class for viewAsChainedAllocs that takes either ReadHandle or // WriteHandle @@ -1657,29 +1675,12 @@ class CacheAllocator : public CacheBase { template folly::IOBuf convertToIOBufT(Handle& handle); - // Moves a chained item to a different slab. This should only be used during - // slab release after the item's exclusive bit has been set. The user supplied - // callback is responsible for copying the contents and fixing the semantics - // of chained item. - // - // Note: If we have successfully moved the old item into the new, the - // newItemHdl is reset and no longer usable by the caller. - // - // @param oldItem Reference to the item being moved - // @param newItemHdl Reference to the handle of the new item being - // moved into - // - // @return true If the move was completed, and the containers were updated - // successfully. - bool moveChainedItem(ChainedItem& oldItem, WriteHandle& newItemHdl); - // Transfers the chain ownership from parent to newParent. Parent // will be unmarked as having chained allocations. Parent will not be null // after calling this API. // - // Parent and NewParent must be valid handles to items with same key and - // parent must have chained items and parent handle must be the only - // outstanding handle for parent. New parent must be without any chained item + // NewParent must be valid handles to item with same key as Parent and + // Parent must have chained items. New parent must be without any chained item // handles. // // Chained item lock for the parent's key needs to be held in exclusive mode. @@ -1688,7 +1689,7 @@ class CacheAllocator : public CacheBase { // @param newParent the new parent for the chain // // @throw if any of the conditions for parent or newParent are not met. - void transferChainLocked(WriteHandle& parent, WriteHandle& newParent); + void transferChainLocked(Item& parent, WriteHandle& newParent); // replace a chained item in the existing chain. This needs to be called // with the chained item lock held exclusive @@ -1702,6 +1703,24 @@ class CacheAllocator : public CacheBase { WriteHandle newItemHdl, const Item& parent); + // + // Performs the actual inplace replace - it is called from + // moveChainedItem and replaceChainedItemLocked + // must hold chainedItemLock + // + // @param oldItem the item we are replacing in the chain + // @param newItem the item we are replacing it with + // @param parent the parent for the chain + // @param fromMove used to determine if the replaced was called from + // moveChainedItem - we avoid the handle destructor + // in this case. + // + // @return handle to the oldItem + void replaceInChainLocked(Item& oldItem, + WriteHandle& newItemHdl, + const Item& parent, + bool fromMove); + // Insert an item into MM container. The caller must hold a valid handle for // the item. // @@ -2016,7 +2035,7 @@ auto& mmContainer = getMMContainer(tid, pid, cid); throw std::runtime_error("Not supported for chained items"); } - if (candidate->markMoving(true)) { + if (candidate->markMoving()) { mmContainer.remove(itr); candidates.push_back(candidate); } else { @@ -2089,7 +2108,7 @@ auto& mmContainer = getMMContainer(tid, pid, cid); // TODO: only allow it for read-only items? // or implement mvcc - if (candidate->markMoving(true)) { + if (candidate->markMoving()) { // promotions should rarely fail since we already marked moving mmContainer.remove(itr); candidates.push_back(candidate); diff --git a/cachelib/allocator/CacheItem-inl.h b/cachelib/allocator/CacheItem-inl.h index b3585f7a35..bf77b43aa5 100644 --- a/cachelib/allocator/CacheItem-inl.h +++ b/cachelib/allocator/CacheItem-inl.h @@ -238,8 +238,8 @@ bool CacheItem::markForEvictionWhenMoving() { } template -bool CacheItem::markMoving(bool failIfRefNotZero) { - return ref_.markMoving(failIfRefNotZero); +bool CacheItem::markMoving() { + return ref_.markMoving(); } template diff --git a/cachelib/allocator/CacheItem.h b/cachelib/allocator/CacheItem.h index 6728b654eb..b7ae24fe6b 100644 --- a/cachelib/allocator/CacheItem.h +++ b/cachelib/allocator/CacheItem.h @@ -312,9 +312,9 @@ class CACHELIB_PACKED_ATTR CacheItem { // // @return true on success, failure if item is marked as exclusive // @throw exception::RefcountOverflow on ref count overflow - FOLLY_ALWAYS_INLINE RefcountWithFlags::incResult incRef(bool failIfMoving) { + FOLLY_ALWAYS_INLINE RefcountWithFlags::incResult incRef() { try { - return ref_.incRef(failIfMoving); + return ref_.incRef(); } catch (exception::RefcountOverflow& e) { throw exception::RefcountOverflow( folly::sformat("{} item: {}", e.what(), toString())); @@ -381,7 +381,7 @@ class CACHELIB_PACKED_ATTR CacheItem { * Unmarking moving will also return the refcount at the moment of * unmarking. */ - bool markMoving(bool failIfRefNotZero); + bool markMoving(); RefcountWithFlags::Value unmarkMoving() noexcept; bool isMoving() const noexcept; bool isOnlyMoving() const noexcept; diff --git a/cachelib/allocator/MM2Q-inl.h b/cachelib/allocator/MM2Q-inl.h index 07aae775f7..7f00b96131 100644 --- a/cachelib/allocator/MM2Q-inl.h +++ b/cachelib/allocator/MM2Q-inl.h @@ -247,6 +247,12 @@ MM2Q::Container::getEvictionIterator() const noexcept { return LockedIterator{std::move(l), lru_.rbegin()}; } +template T::*HookPtr> +template +void MM2Q::Container::withContainerLock(F&& fun) { + lruMutex_->lock_combine([this, &fun]() { fun(); }); +} + template T::*HookPtr> template void MM2Q::Container::withEvictionIterator(F&& fun) { diff --git a/cachelib/allocator/MM2Q.h b/cachelib/allocator/MM2Q.h index 62b644f9c6..292a612cde 100644 --- a/cachelib/allocator/MM2Q.h +++ b/cachelib/allocator/MM2Q.h @@ -503,6 +503,10 @@ class MM2Q { template void withEvictionIterator(F&& f); + // Execute provided function under container lock. + template + void withContainerLock(F&& f); + // Execute provided function under container lock. Function gets // iterator passed as parameter. template diff --git a/cachelib/allocator/MMLru-inl.h b/cachelib/allocator/MMLru-inl.h index 751bcca5c1..427753f853 100644 --- a/cachelib/allocator/MMLru-inl.h +++ b/cachelib/allocator/MMLru-inl.h @@ -218,6 +218,12 @@ MMLru::Container::getEvictionIterator() const noexcept { return LockedIterator{std::move(l), lru_.rbegin()}; } +template T::*HookPtr> +template +void MMLru::Container::withContainerLock(F&& fun) { + lruMutex_->lock_combine([this, &fun]() { fun(); }); +} + template T::*HookPtr> template void MMLru::Container::withEvictionIterator(F&& fun) { diff --git a/cachelib/allocator/MMLru.h b/cachelib/allocator/MMLru.h index cf3253349a..ad6c4b784b 100644 --- a/cachelib/allocator/MMLru.h +++ b/cachelib/allocator/MMLru.h @@ -377,6 +377,10 @@ class MMLru { template void withEvictionIterator(F&& f); + // Execute provided function under container lock. + template + void withContainerLock(F&& f); + template void withPromotionIterator(F&& f); diff --git a/cachelib/allocator/MMTinyLFU-inl.h b/cachelib/allocator/MMTinyLFU-inl.h index 9203a54dd6..46a760909d 100644 --- a/cachelib/allocator/MMTinyLFU-inl.h +++ b/cachelib/allocator/MMTinyLFU-inl.h @@ -220,6 +220,13 @@ MMTinyLFU::Container::getEvictionIterator() const noexcept { return LockedIterator{std::move(l), *this}; } +template T::*HookPtr> +template +void MMTinyLFU::Container::withContainerLock(F&& fun) { + LockHolder l(lruMutex_); + fun(); +} + template T::*HookPtr> template void MMTinyLFU::Container::withEvictionIterator(F&& fun) { diff --git a/cachelib/allocator/MMTinyLFU.h b/cachelib/allocator/MMTinyLFU.h index eb45cefd22..a0d4386b83 100644 --- a/cachelib/allocator/MMTinyLFU.h +++ b/cachelib/allocator/MMTinyLFU.h @@ -497,6 +497,10 @@ class MMTinyLFU { template void withEvictionIterator(F&& f); + // Execute provided function under container lock. + template + void withContainerLock(F&& f); + template void withPromotionIterator(F&& f); diff --git a/cachelib/allocator/Refcount.h b/cachelib/allocator/Refcount.h index 902f217760..b254093d07 100644 --- a/cachelib/allocator/Refcount.h +++ b/cachelib/allocator/Refcount.h @@ -140,9 +140,9 @@ class FOLLY_PACK_ATTR RefcountWithFlags { // @return true if refcount is bumped. false otherwise (if item is exclusive) // @throw exception::RefcountOverflow if new count would be greater than // maxCount - FOLLY_ALWAYS_INLINE incResult incRef(bool failIfMoving) { + FOLLY_ALWAYS_INLINE incResult incRef() { incResult res = incOk; - auto predicate = [failIfMoving, &res](const Value curValue) { + auto predicate = [&res](const Value curValue) { Value bitMask = getAdminRef(); const bool exlusiveBitIsSet = curValue & bitMask; @@ -151,7 +151,7 @@ class FOLLY_PACK_ATTR RefcountWithFlags { } else if (exlusiveBitIsSet && (curValue & kAccessRefMask) == 0) { res = incFailedEviction; return false; - } else if (exlusiveBitIsSet && failIfMoving) { + } else if (exlusiveBitIsSet) { res = incFailedMoving; return false; } @@ -330,14 +330,18 @@ class FOLLY_PACK_ATTR RefcountWithFlags { * Unmarking clears `kExclusive` bit and decreses the interanl refCount by 1. * `unmarkMoving` does does not depend on `isInMMContainer` */ - bool markMoving(bool failIfRefNotZero) { + bool markMoving() { Value linkedBitMask = getAdminRef(); Value exclusiveBitMask = getAdminRef(); + Value isChainedItemFlag = getFlag(); - auto predicate = [failIfRefNotZero, linkedBitMask, exclusiveBitMask](const Value curValue) { + auto predicate = [linkedBitMask, exclusiveBitMask, isChainedItemFlag](const Value curValue) { const bool unlinked = !(curValue & linkedBitMask); const bool alreadyExclusive = curValue & exclusiveBitMask; - if (failIfRefNotZero && (curValue & kAccessRefMask) != 0) { + const bool isChained = curValue & isChainedItemFlag; + + // chained item can have ref count == 1, this just means it's linked in the chain + if ((curValue & kAccessRefMask) > isChained ? 1 : 0) { return false; } if (unlinked || alreadyExclusive) { diff --git a/cachelib/allocator/tests/AllocatorTypeTest.cpp b/cachelib/allocator/tests/AllocatorTypeTest.cpp index a572485e8d..ad38588bcb 100644 --- a/cachelib/allocator/tests/AllocatorTypeTest.cpp +++ b/cachelib/allocator/tests/AllocatorTypeTest.cpp @@ -288,8 +288,8 @@ TYPED_TEST(BaseAllocatorTest, AddChainedItemMultiThreadWithMovingAndSync) { this->testAddChainedItemMultithreadWithMovingAndSync(); } -TYPED_TEST(BaseAllocatorTest, TransferChainWhileMoving) { - this->testTransferChainWhileMoving(); +TYPED_TEST(BaseAllocatorTest, TransferChainAfterMoving) { + this->testTransferChainAfterMoving(); } TYPED_TEST(BaseAllocatorTest, AddAndPopChainedItemMultithread) { diff --git a/cachelib/allocator/tests/BaseAllocatorTest.h b/cachelib/allocator/tests/BaseAllocatorTest.h index e059972eb4..e7bf0db8b6 100644 --- a/cachelib/allocator/tests/BaseAllocatorTest.h +++ b/cachelib/allocator/tests/BaseAllocatorTest.h @@ -3655,6 +3655,16 @@ class BaseAllocatorTest : public AllocatorTest { sourceAlloc); otherThread.join(); + // in our new version with marking item as moving, move attempts + // will only fail if there is a concurrent set to that item, in + // this case if the handle to an item is held, the slab release + // will keep trying to mark the item as moving - we currently + // don't have a counter for that (but this test assumes that + // if handle is held then moveForSlabRelease will retry, + // that is where the move attempts counter is incremented) + // + // as a fix, we increment the move attempts counter during + // markMovingForSlabRelase too XLOG(INFO, "Number of move retry attempts: ", allocator.getSlabReleaseStats().numMoveAttempts); ASSERT_GT(allocator.getSlabReleaseStats().numMoveAttempts, 1); @@ -4832,7 +4842,7 @@ class BaseAllocatorTest : public AllocatorTest { std::memcpy(newItem.getMemory(), oldItem.getMemory(), oldItem.getSize()); ++numMoves; - }); + }, {}, 1000000 /* lots of moving tries */); AllocatorT alloc(config); const size_t numBytes = alloc.getCacheMemoryStats().ramCacheSize; @@ -4873,7 +4883,7 @@ class BaseAllocatorTest : public AllocatorTest { } /* sleep override */ - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); } }; @@ -4881,7 +4891,7 @@ class BaseAllocatorTest : public AllocatorTest { auto releaseFn = [&] { for (unsigned int i = 0; i < 5;) { /* sleep override */ - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); ClassId cid = static_cast(i); alloc.releaseSlab(pid, cid, SlabReleaseMode::kRebalance); @@ -5098,7 +5108,7 @@ class BaseAllocatorTest : public AllocatorTest { auto releaseFn = [&] { for (unsigned int i = 0; i < 5;) { /* sleep override */ - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); ClassId cid = static_cast(i); alloc.releaseSlab(pid, cid, SlabReleaseMode::kRebalance); @@ -5157,9 +5167,10 @@ class BaseAllocatorTest : public AllocatorTest { lookupFn("yolo"); } - // while a chained item could be moved, try to transfer its parent and - // validate that move succeeds correctly. - void testTransferChainWhileMoving() { + // while a chained item could be moved - it is sync on parent moving bit. + // try to transfer its parent after we moved and + // validate that transfer succeeds correctly. + void testTransferChainAfterMoving() { // create an allocator worth 10 slabs. typename AllocatorT::Config config; config.configureChainedItems(); @@ -5180,15 +5191,13 @@ class BaseAllocatorTest : public AllocatorTest { struct TestSyncObj : public AllocatorT::SyncObj { TestSyncObj(std::mutex& m, std::atomic& firstTime, - folly::Baton<>& startedMoving, - folly::Baton<>& changedParent) + folly::Baton<>& startedMoving) : l(m) { if (!firstTime) { return; } firstTime = false; startedMoving.post(); - changedParent.wait(); } std::lock_guard l; @@ -5201,9 +5210,6 @@ class BaseAllocatorTest : public AllocatorTest { // baton to indicate that the move process has started so that we can // switch the parent folly::Baton<> startedMoving; - // baton to indicate that the parent has been switched so that the move - // process can proceed - folly::Baton<> changedParent; const size_t numMovingAttempts = 100; std::atomic numMoves{0}; @@ -5215,11 +5221,10 @@ class BaseAllocatorTest : public AllocatorTest { oldItem.getSize()); ++numMoves; }, - [&m, &startedMoving, &changedParent, - &firstTimeMovingSync](typename Item::Key key) { + [&m, &startedMoving, &firstTimeMovingSync](typename Item::Key key) { XLOG(ERR) << "Moving" << key; return std::make_unique(m, firstTimeMovingSync, - startedMoving, changedParent); + startedMoving); }, numMovingAttempts); @@ -5249,24 +5254,19 @@ class BaseAllocatorTest : public AllocatorTest { auto slabRelease = std::async(releaseFn); startedMoving.wait(); + // wait for slab release to complete. + slabRelease.wait(); // we know moving sync is held now. { auto newParent = alloc.allocate(pid, movingKey, 600); - auto parent = alloc.findToWrite(movingKey); + auto parent = alloc.findToWrite(movingKey); //parent is marked moving during moved, once finished we will get handle alloc.transferChainAndReplace(parent, newParent); } - // indicate that we changed the parent. This should abort the current - // moving attempt, re-allocate the item and eventually succeed in moving. - changedParent.post(); - - // wait for slab release to complete. - slabRelease.wait(); - EXPECT_EQ(numMoves, 1); auto slabReleaseStats = alloc.getSlabReleaseStats(); - EXPECT_EQ(slabReleaseStats.numMoveAttempts, 2); + EXPECT_EQ(slabReleaseStats.numMoveAttempts, 1); EXPECT_EQ(slabReleaseStats.numMoveSuccesses, 1); auto handle = alloc.find(movingKey); @@ -5980,7 +5980,6 @@ class BaseAllocatorTest : public AllocatorTest { EXPECT_EQ(nullptr, util::allocateAccessible(alloc, poolId, "large", largeSize)); - std::this_thread::sleep_for(std::chrono::seconds{1}); // trigger the slab rebalance EXPECT_EQ(nullptr, util::allocateAccessible(alloc, poolId, "large", largeSize)); diff --git a/cachelib/allocator/tests/ItemTest.cpp b/cachelib/allocator/tests/ItemTest.cpp index 2f25cc07f0..54bac1945a 100644 --- a/cachelib/allocator/tests/ItemTest.cpp +++ b/cachelib/allocator/tests/ItemTest.cpp @@ -85,7 +85,7 @@ TEST(ItemTest, ExpiryTime) { // So that exclusive bit will be set item->markAccessible(); // Test that writes fail while the item is moving - result = item->markMoving(true); + result = item->markMoving(); EXPECT_TRUE(result); result = item->updateExpiryTime(0); EXPECT_FALSE(result); diff --git a/cachelib/allocator/tests/RebalanceStrategyTest.cpp b/cachelib/allocator/tests/RebalanceStrategyTest.cpp index cb5c4cfd51..849483892d 100644 --- a/cachelib/allocator/tests/RebalanceStrategyTest.cpp +++ b/cachelib/allocator/tests/RebalanceStrategyTest.cpp @@ -214,6 +214,9 @@ class RebalanceStrategyTest : public testing::Test { config.poolRebalancerFreeAllocThreshold = 20; initAllocatorConfigForStrategy(config, LruTailAge); + //TODO: why does this fail with orig. value of 8? + //on upstream this fails too, it always reports 4 instead + //of the original test value, which is 8 expected slabs doWork(config, true, 8); } diff --git a/cachelib/allocator/tests/RefCountTest.cpp b/cachelib/allocator/tests/RefCountTest.cpp index 8f8ae9609a..f26f76aa4d 100644 --- a/cachelib/allocator/tests/RefCountTest.cpp +++ b/cachelib/allocator/tests/RefCountTest.cpp @@ -52,7 +52,7 @@ void RefCountTest::testMultiThreaded() { nLocalRef--; ref.markAccessible(); } else { - ref.incRef(true); + ref.incRef(); nLocalRef++; ref.unmarkAccessible(); } @@ -101,12 +101,12 @@ void RefCountTest::testBasic() { ASSERT_FALSE(ref.template isFlagSet()); for (uint32_t i = 0; i < RefcountWithFlags::kAccessRefMask; i++) { - ASSERT_EQ(ref.incRef(true),RefcountWithFlags::incOk); + ASSERT_EQ(ref.incRef(),RefcountWithFlags::incOk); } // Incrementing past the max will fail auto rawRef = ref.getRaw(); - ASSERT_THROW(ref.incRef(true), std::overflow_error); + ASSERT_THROW(ref.incRef(), std::overflow_error); ASSERT_EQ(rawRef, ref.getRaw()); // Bumping up access ref shouldn't affect admin ref and flags @@ -152,11 +152,11 @@ void RefCountTest::testBasic() { ASSERT_FALSE(ref.template isFlagSet()); // conditionally set flags - ASSERT_FALSE(ref.markMoving(true)); + ASSERT_FALSE(ref.markMoving()); ref.markInMMContainer(); // only first one succeeds - ASSERT_TRUE(ref.markMoving(true)); - ASSERT_FALSE(ref.markMoving(true)); + ASSERT_TRUE(ref.markMoving()); + ASSERT_FALSE(ref.markMoving()); ref.unmarkInMMContainer(); ref.template setFlag(); @@ -193,7 +193,7 @@ void RefCountTest::testMarkForEvictionAndMoving() { RefcountWithFlags ref; ref.markInMMContainer(); - ASSERT_TRUE(ref.markMoving(true)); + ASSERT_TRUE(ref.markMoving()); ASSERT_FALSE(ref.markForEviction()); ref.unmarkInMMContainer(); @@ -207,33 +207,19 @@ void RefCountTest::testMarkForEvictionAndMoving() { ref.markInMMContainer(); ASSERT_TRUE(ref.markForEviction()); - ASSERT_FALSE(ref.markMoving(true)); + ASSERT_FALSE(ref.markMoving()); ref.unmarkInMMContainer(); auto ret = ref.unmarkForEviction(); ASSERT_EQ(ret, 0); } - { - // can mark moving when ref count > 0 - RefcountWithFlags ref; - ref.markInMMContainer(); - - ref.incRef(true); - - ASSERT_TRUE(ref.markMoving(false)); - - ref.unmarkInMMContainer(); - auto ret = ref.unmarkMoving(); - ASSERT_EQ(ret, 1); - } - { // cannot mark for eviction when ref count > 0 RefcountWithFlags ref; ref.markInMMContainer(); - ref.incRef(true); + ref.incRef(); ASSERT_FALSE(ref.markForEviction()); } } diff --git a/cachelib/allocator/tests/SimpleRebalancingTest.h b/cachelib/allocator/tests/SimpleRebalancingTest.h index 634882c730..3f1869ede3 100644 --- a/cachelib/allocator/tests/SimpleRebalancingTest.h +++ b/cachelib/allocator/tests/SimpleRebalancingTest.h @@ -104,7 +104,7 @@ class SimpleRebalanceTest : public testing::Test { // Sleep for 2 seconds to let the rebalancing work /* sleep override */ - std::this_thread::sleep_for(std::chrono::seconds(3)); + std::this_thread::sleep_for(std::chrono::seconds(10)); // Evicted keys shouldn't be in the allocator anymore ASSERT_FALSE(evictedKeys.empty()); diff --git a/cachelib/cachebench/runner/CacheStressor.h b/cachelib/cachebench/runner/CacheStressor.h index b222fa421f..41090c93a3 100644 --- a/cachelib/cachebench/runner/CacheStressor.h +++ b/cachelib/cachebench/runner/CacheStressor.h @@ -77,7 +77,7 @@ class CacheStressor : public Stressor { std::unique_lock lock; CacheStressSyncObj(CacheStressor& s, std::string itemKey) - : lock{s.chainedItemAcquireUniqueLock(itemKey)} {} + : lock{s.chainedItemTryAcquireUniqueLock(itemKey)} {} }; movingSync = [this](typename CacheT::Item::Key key) { return std::make_unique(*this, key.str()); @@ -247,6 +247,10 @@ class CacheStressor : public Stressor { using Lock = std::unique_lock; return lockEnabled_ ? Lock{getLock(key)} : Lock{}; } + auto chainedItemTryAcquireUniqueLock(Key key) { + using Lock = std::unique_lock; + return lockEnabled_ ? Lock{getLock(key), std::try_to_lock} : Lock{}; + } // populate the input item handle according to the stress setup. void populateItem(WriteHandle& handle, const std::string& itemValue = "") { diff --git a/cachelib/cachebench/test_configs/small_moving_bg.json b/cachelib/cachebench/test_configs/small_moving_bg.json new file mode 100644 index 0000000000..c4838f42b5 --- /dev/null +++ b/cachelib/cachebench/test_configs/small_moving_bg.json @@ -0,0 +1,35 @@ +// @nolint like default.json, but moves items during slab release instead of evicting them. +{ + "cache_config" : { + "cacheSizeMB" : 2248, + "cacheDir": "/tmp/mem-tier5", + "memoryTiers" : [ + { + "ratio": 1, + "memBindNodes": 0 + }, { + "ratio": 1, + "memBindNodes": 0 + } + ], + "poolRebalanceIntervalSec" : 1, + "moveOnSlabRelease" : true, + "rebalanceMinSlabs" : 2, + "evictorThreads": 2, + "promoterThreads": 2 + }, + "test_config" : + { + "preallocateCache" : true, + "numOps" : 20000000, + "numThreads" : 32, + "numKeys" : 250000, + "generator": "online", + "keySizeRange" : [1, 8, 32, 64, 128, 256, 512], + "keySizeRangeProbability" : [0.1, 0.1, 0.2, 0.2, 0.3, 0.1], + "valSizeRange" : [1, 128, 512, 1024, 4096, 10240, 20480, 40960, 60000], + "valSizeRangeProbability" : [0.1, 0.1, 0.1, 0.2, 0.2, 0.1, 0.1, 0.1], + "getRatio" : 0.70, + "setRatio" : 0.30 + } + } diff --git a/cachelib/common/Mutex.h b/cachelib/common/Mutex.h index 1d6e5898f1..15b440d406 100644 --- a/cachelib/common/Mutex.h +++ b/cachelib/common/Mutex.h @@ -341,6 +341,7 @@ class RWBucketLocks : public BaseBucketLocks { using Lock = LockType; using ReadLockHolder = ReadLockHolderType; using WriteLockHolder = WriteLockHolderType; + using LockHolder = std::unique_lock; RWBucketLocks(uint32_t locksPower, std::shared_ptr hasher) : Base::BaseBucketLocks(locksPower, std::move(hasher)) {} @@ -357,6 +358,11 @@ class RWBucketLocks : public BaseBucketLocks { return WriteLockHolder{Base::getLock(args...)}; } + template + LockHolder tryLockExclusive(Args... args) noexcept { + return LockHolder(Base::getLock(args...), std::try_to_lock); + } + // try to grab the reader lock for a limit _timeout_ duration template ReadLockHolder lockShared(const std::chrono::microseconds& timeout, diff --git a/run_tests.sh b/run_tests.sh index e575dbc62a..6ff2ac65ed 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -13,3 +13,4 @@ fi ../bin/cachebench --json_test_config ../test_configs/consistency/navy.json ../bin/cachebench --json_test_config ../test_configs/consistency/navy-multi-tier.json +../bin/cachebench --json_test_config ../test_configs/small_moving_bg.json From 08d8f331d808fe7af1a91a26173d4b5a446594cb Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Mon, 24 Jul 2023 14:26:23 -0700 Subject: [PATCH 27/31] edit dockerfile --- docker/images/centos-8streams.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/images/centos-8streams.Dockerfile b/docker/images/centos-8streams.Dockerfile index b916ab760c..e0c31226a1 100644 --- a/docker/images/centos-8streams.Dockerfile +++ b/docker/images/centos-8streams.Dockerfile @@ -17,6 +17,8 @@ json-c-devel \ perf \ numactl +# updated to fix compile errors and better symbol +# resolving in VTune RUN dnf -y install gcc-toolset-12 RUN echo "source /opt/rh/gcc-toolset-12/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] From 316133cd6b9f81275918af86149c2d01a23dc5f5 Mon Sep 17 00:00:00 2001 From: Daniel Byrne Date: Tue, 25 Jul 2023 14:41:46 -0700 Subject: [PATCH 28/31] these submodules work --- cachelib/external/fbthrift | 2 +- cachelib/external/fizz | 2 +- cachelib/external/folly | 2 +- cachelib/external/wangle | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cachelib/external/fbthrift b/cachelib/external/fbthrift index fb3c6ce37a..cbc3de581f 160000 --- a/cachelib/external/fbthrift +++ b/cachelib/external/fbthrift @@ -1 +1 @@ -Subproject commit fb3c6ce37aab5aecbb39c827e0ae84256c64a44b +Subproject commit cbc3de581fdf36ba474b0c135b9e785e504f1c1e diff --git a/cachelib/external/fizz b/cachelib/external/fizz index 5551610370..80ba4b64d1 160000 --- a/cachelib/external/fizz +++ b/cachelib/external/fizz @@ -1 +1 @@ -Subproject commit 555161037025db59658ae5d0277c4c3e1e49817e +Subproject commit 80ba4b64d1138025a3f61e4cd3c826405cd9e8cb diff --git a/cachelib/external/folly b/cachelib/external/folly index 017e426621..ce2b95715d 160000 --- a/cachelib/external/folly +++ b/cachelib/external/folly @@ -1 +1 @@ -Subproject commit 017e42662179411f83eb24c7100b3af7f8a61518 +Subproject commit ce2b95715de229fcb51bd97410469a3ad4d2bfb2 diff --git a/cachelib/external/wangle b/cachelib/external/wangle index 68b1ec08f2..44690e7894 160000 --- a/cachelib/external/wangle +++ b/cachelib/external/wangle @@ -1 +1 @@ -Subproject commit 68b1ec08f23196e0ad1dd2dfbb2308c095caf440 +Subproject commit 44690e7894842a7127245837b69627d4b964aabd From 8d2c390a327fe434ea1b60e34ce03f4cd777ed7a Mon Sep 17 00:00:00 2001 From: Sounak Gupta Date: Fri, 28 Jul 2023 01:39:04 -0700 Subject: [PATCH 29/31] Track latency of per item eviction/promotion between memory tiers --- cachelib/allocator/Cache.cpp | 4 ++++ cachelib/allocator/CacheAllocator-inl.h | 7 +++---- cachelib/allocator/CacheAllocator.h | 4 +++- cachelib/allocator/CacheStats.cpp | 2 ++ cachelib/allocator/CacheStats.h | 2 ++ cachelib/allocator/CacheStatsInternal.h | 2 ++ cachelib/cachebench/cache/Cache-inl.h | 2 ++ cachelib/cachebench/cache/CacheStats.h | 6 ++++++ cachelib/common/PercentileStats.h | 11 ++++++----- 9 files changed, 30 insertions(+), 10 deletions(-) diff --git a/cachelib/allocator/Cache.cpp b/cachelib/allocator/Cache.cpp index b871a04782..37457cc3e9 100644 --- a/cachelib/allocator/Cache.cpp +++ b/cachelib/allocator/Cache.cpp @@ -478,6 +478,10 @@ void CacheBase::updateGlobalCacheStats(const std::string& statPrefix) const { visitEstimates(uploadStatsNanoToMicro, stats.allocateLatencyNs, statPrefix + "allocate.latency_us"); + visitEstimates(uploadStatsNanoToMicro, stats.bgEvictLatencyNs, + statPrefix + "background.eviction.latency_us"); + visitEstimates(uploadStatsNanoToMicro, stats.bgPromoteLatencyNs, + statPrefix + "background.promotion.latency_us"); visitEstimates(uploadStatsNanoToMicro, stats.moveChainedLatencyNs, statPrefix + "move.chained.latency_us"); visitEstimates(uploadStatsNanoToMicro, stats.moveRegularLatencyNs, diff --git a/cachelib/allocator/CacheAllocator-inl.h b/cachelib/allocator/CacheAllocator-inl.h index a898d9b4fd..11e9058a34 100644 --- a/cachelib/allocator/CacheAllocator-inl.h +++ b/cachelib/allocator/CacheAllocator-inl.h @@ -426,8 +426,7 @@ CacheAllocator::allocateInternalTier(TierId tid, uint32_t expiryTime, bool fromBgThread, bool evict) { - util::LatencyTracker tracker{stats().allocateLatency_}; - + util::LatencyTracker tracker{stats().allocateLatency_, static_cast(!fromBgThread)}; SCOPE_FAIL { stats_.invalidAllocs.inc(); }; // number of bytes required for this item @@ -435,8 +434,8 @@ CacheAllocator::allocateInternalTier(TierId tid, // the allocation class in our memory allocator. const auto cid = allocator_[tid]->getAllocationClassId(pid, requiredSize); - util::RollingLatencyTracker rollTracker{ - (*stats_.classAllocLatency)[tid][pid][cid]}; + + util::RollingLatencyTracker rollTracker{(*stats_.classAllocLatency)[tid][pid][cid]}; (*stats_.allocAttempts)[tid][pid][cid].inc(); diff --git a/cachelib/allocator/CacheAllocator.h b/cachelib/allocator/CacheAllocator.h index 0ec7224603..b3201daf76 100644 --- a/cachelib/allocator/CacheAllocator.h +++ b/cachelib/allocator/CacheAllocator.h @@ -2016,7 +2016,8 @@ class CacheAllocator : public CacheBase { // exposed for the background evictor to iterate through the memory and evict // in batch. This should improve insertion path for tiered memory config size_t traverseAndEvictItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { -auto& mmContainer = getMMContainer(tid, pid, cid); + util::LatencyTracker tracker{stats().bgEvictLatency_, batch}; + auto& mmContainer = getMMContainer(tid, pid, cid); size_t evictions = 0; size_t evictionCandidates = 0; std::vector candidates; @@ -2089,6 +2090,7 @@ auto& mmContainer = getMMContainer(tid, pid, cid); } size_t traverseAndPromoteItems(unsigned int tid, unsigned int pid, unsigned int cid, size_t batch) { + util::LatencyTracker tracker{stats().bgPromoteLatency_, batch}; auto& mmContainer = getMMContainer(tid, pid, cid); size_t promotions = 0; std::vector candidates; diff --git a/cachelib/allocator/CacheStats.cpp b/cachelib/allocator/CacheStats.cpp index a5dd962dc2..453d3a0abb 100644 --- a/cachelib/allocator/CacheStats.cpp +++ b/cachelib/allocator/CacheStats.cpp @@ -106,6 +106,8 @@ void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { ret.numNvmItemDestructorAllocErrors = numNvmItemDestructorAllocErrors.get(); ret.allocateLatencyNs = this->allocateLatency_.estimate(); + ret.bgEvictLatencyNs = this->bgEvictLatency_.estimate(); + ret.bgPromoteLatencyNs = this->bgPromoteLatency_.estimate(); ret.moveChainedLatencyNs = this->moveChainedLatency_.estimate(); ret.moveRegularLatencyNs = this->moveRegularLatency_.estimate(); ret.nvmLookupLatencyNs = this->nvmLookupLatency_.estimate(); diff --git a/cachelib/allocator/CacheStats.h b/cachelib/allocator/CacheStats.h index 48ea90f6b8..cda2690bf8 100644 --- a/cachelib/allocator/CacheStats.h +++ b/cachelib/allocator/CacheStats.h @@ -510,6 +510,8 @@ struct GlobalCacheStats { // latency and percentile stats of various cachelib operations util::PercentileStats::Estimates allocateLatencyNs{}; + util::PercentileStats::Estimates bgEvictLatencyNs{}; + util::PercentileStats::Estimates bgPromoteLatencyNs{}; util::PercentileStats::Estimates moveChainedLatencyNs{}; util::PercentileStats::Estimates moveRegularLatencyNs{}; util::PercentileStats::Estimates nvmLookupLatencyNs{}; diff --git a/cachelib/allocator/CacheStatsInternal.h b/cachelib/allocator/CacheStatsInternal.h index ef41ea7bbc..da48df2d8f 100644 --- a/cachelib/allocator/CacheStatsInternal.h +++ b/cachelib/allocator/CacheStatsInternal.h @@ -189,6 +189,8 @@ struct Stats { // latency stats of various cachelib operations mutable util::PercentileStats allocateLatency_; + mutable util::PercentileStats bgEvictLatency_; + mutable util::PercentileStats bgPromoteLatency_; mutable util::PercentileStats moveChainedLatency_; mutable util::PercentileStats moveRegularLatency_; mutable util::PercentileStats nvmLookupLatency_; diff --git a/cachelib/cachebench/cache/Cache-inl.h b/cachelib/cachebench/cache/Cache-inl.h index 23481ad099..f1c5248718 100644 --- a/cachelib/cachebench/cache/Cache-inl.h +++ b/cachelib/cachebench/cache/Cache-inl.h @@ -743,6 +743,8 @@ Stats Cache::getStats() const { static_cast(itemRecords_.count()) - totalDestructor_; ret.cacheAllocateLatencyNs = cacheStats.allocateLatencyNs; + ret.cacheBgEvictLatencyNs = cacheStats.bgEvictLatencyNs; + ret.cacheBgPromoteLatencyNs = cacheStats.bgPromoteLatencyNs; ret.cacheFindLatencyNs = cacheFindLatency_.estimate(); // Populate counters. diff --git a/cachelib/cachebench/cache/CacheStats.h b/cachelib/cachebench/cache/CacheStats.h index eecb89bb96..72d71f4faa 100644 --- a/cachelib/cachebench/cache/CacheStats.h +++ b/cachelib/cachebench/cache/CacheStats.h @@ -91,6 +91,8 @@ struct Stats { uint64_t numNvmItemRemovedSetSize{0}; util::PercentileStats::Estimates cacheAllocateLatencyNs; + util::PercentileStats::Estimates cacheBgEvictLatencyNs; + util::PercentileStats::Estimates cacheBgPromoteLatencyNs; util::PercentileStats::Estimates cacheFindLatencyNs; double nvmReadLatencyMicrosP50{0}; @@ -282,6 +284,8 @@ struct Stats { printLatencies("Cache Find API latency", cacheFindLatencyNs); printLatencies("Cache Allocate API latency", cacheAllocateLatencyNs); + printLatencies("Cache Background Eviction API latency", cacheBgEvictLatencyNs); + printLatencies("Cache Background Promotion API latency", cacheBgPromoteLatencyNs); } } @@ -510,6 +514,8 @@ struct Stats { counters["find_latency_p99"] = cacheFindLatencyNs.p99; counters["alloc_latency_p99"] = cacheAllocateLatencyNs.p99; + counters["bg_evict_latency_p99"] = cacheBgEvictLatencyNs.p99; + counters["bg_promote_latency_p99"] = cacheBgPromoteLatencyNs.p99; counters["ram_hit_rate"] = calcInvertPctFn(numCacheGetMiss, numCacheGets); counters["nvm_hit_rate"] = calcInvertPctFn(numCacheGetMiss, numCacheGets); diff --git a/cachelib/common/PercentileStats.h b/cachelib/common/PercentileStats.h index bdd3255eba..c308671ee9 100644 --- a/cachelib/common/PercentileStats.h +++ b/cachelib/common/PercentileStats.h @@ -107,16 +107,16 @@ class PercentileStats { class LatencyTracker { public: - explicit LatencyTracker(PercentileStats& stats) - : stats_(&stats), begin_(std::chrono::steady_clock::now()) {} + explicit LatencyTracker(PercentileStats& stats, size_t nSamples = 1) + : stats_(&stats), nSamples_(nSamples), begin_(std::chrono::steady_clock::now()) {} LatencyTracker() {} ~LatencyTracker() { - if (stats_) { + if (nSamples_ > 0 && stats_) { auto tp = std::chrono::steady_clock::now(); auto diffNanos = std::chrono::duration_cast(tp - begin_) .count(); - stats_->trackValue(static_cast(diffNanos), tp); + stats_->trackValue(static_cast(diffNanos/nSamples_), tp); } } @@ -124,7 +124,7 @@ class LatencyTracker { LatencyTracker& operator=(const LatencyTracker&) = delete; LatencyTracker(LatencyTracker&& rhs) noexcept - : stats_(rhs.stats_), begin_(rhs.begin_) { + : stats_(rhs.stats_), nSamples_(rhs.nSamples_), begin_(rhs.begin_) { rhs.stats_ = nullptr; } @@ -138,6 +138,7 @@ class LatencyTracker { private: PercentileStats* stats_{nullptr}; + size_t nSamples_{1}; std::chrono::time_point begin_; }; } // namespace util From ff44c3ce6a30e684c57cb413f5c51a26ad3bcb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Chor=C4=85=C5=BCewicz?= Date: Wed, 23 Aug 2023 10:21:21 -0700 Subject: [PATCH 30/31] Update dependencies (#95) * Set dependencies to working versions and use dependencies from build context, instead of downloading cachelib:develop during build step. This makes sure that dependencies are always build in proper versions. * Fix CacheStats size --- .github/workflows/build-cachelib-docker.yml | 1 + cachelib/allocator/CacheStats.cpp | 2 +- cachelib/external/fbthrift | 2 +- cachelib/external/fizz | 2 +- cachelib/external/folly | 2 +- cachelib/external/wangle | 2 +- contrib/build-package.sh | 4 ---- docker/images/build-image.sh | 2 +- docker/images/centos-8streams.Dockerfile | 9 +++++---- docker/images/install-cachelib-deps.sh | 8 +++----- docker/images/install-dsa-deps.sh | 2 +- 11 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-cachelib-docker.yml b/.github/workflows/build-cachelib-docker.yml index be28bc233c..f00c028708 100644 --- a/.github/workflows/build-cachelib-docker.yml +++ b/.github/workflows/build-cachelib-docker.yml @@ -40,6 +40,7 @@ jobs: - name: "checkout sources" uses: actions/checkout@v2 with: + submodules: recursive fetch-depth: 0 - name: Pull the image or rebuild and push it diff --git a/cachelib/allocator/CacheStats.cpp b/cachelib/allocator/CacheStats.cpp index 453d3a0abb..28df383275 100644 --- a/cachelib/allocator/CacheStats.cpp +++ b/cachelib/allocator/CacheStats.cpp @@ -57,7 +57,7 @@ struct SizeVerify {}; void Stats::populateGlobalCacheStats(GlobalCacheStats& ret) const { #ifndef SKIP_SIZE_VERIFY - SizeVerify a = SizeVerify<16192>{}; + SizeVerify a = SizeVerify<16544>{}; std::ignore = a; #endif ret.numCacheGets = numCacheGets.get(); diff --git a/cachelib/external/fbthrift b/cachelib/external/fbthrift index cbc3de581f..a183a5debd 160000 --- a/cachelib/external/fbthrift +++ b/cachelib/external/fbthrift @@ -1 +1 @@ -Subproject commit cbc3de581fdf36ba474b0c135b9e785e504f1c1e +Subproject commit a183a5debdddad9867120bc99e5e0c869299a201 diff --git a/cachelib/external/fizz b/cachelib/external/fizz index 80ba4b64d1..ea53734111 160000 --- a/cachelib/external/fizz +++ b/cachelib/external/fizz @@ -1 +1 @@ -Subproject commit 80ba4b64d1138025a3f61e4cd3c826405cd9e8cb +Subproject commit ea53734111c05685a1981946790cd3244e9d424f diff --git a/cachelib/external/folly b/cachelib/external/folly index ce2b95715d..0d361f7492 160000 --- a/cachelib/external/folly +++ b/cachelib/external/folly @@ -1 +1 @@ -Subproject commit ce2b95715de229fcb51bd97410469a3ad4d2bfb2 +Subproject commit 0d361f7492aab85ee5350bd93c94168cc501ae1d diff --git a/cachelib/external/wangle b/cachelib/external/wangle index 44690e7894..81a74a2932 160000 --- a/cachelib/external/wangle +++ b/cachelib/external/wangle @@ -1 +1 @@ -Subproject commit 44690e7894842a7127245837b69627d4b964aabd +Subproject commit 81a74a29326aacbe5621ddbd2cd4983672d640ef diff --git a/contrib/build-package.sh b/contrib/build-package.sh index f0f3283df0..310792ca2b 100755 --- a/contrib/build-package.sh +++ b/contrib/build-package.sh @@ -197,7 +197,6 @@ case "$1" in folly) NAME=folly SRCDIR=cachelib/external/$NAME - update_submodules=yes cmake_custom_params="-DBUILD_SHARED_LIBS=ON" if test "$build_tests" = "yes" ; then cmake_custom_params="$cmake_custom_params -DBUILD_TESTS=ON" @@ -209,7 +208,6 @@ case "$1" in fizz) NAME=fizz SRCDIR=cachelib/external/$NAME/$NAME - update_submodules=yes cmake_custom_params="-DBUILD_SHARED_LIBS=ON" if test "$build_tests" = "yes" ; then cmake_custom_params="$cmake_custom_params -DBUILD_TESTS=ON" @@ -221,7 +219,6 @@ case "$1" in wangle) NAME=wangle SRCDIR=cachelib/external/$NAME/$NAME - update_submodules=yes cmake_custom_params="-DBUILD_SHARED_LIBS=ON" if test "$build_tests" = "yes" ; then cmake_custom_params="$cmake_custom_params -DBUILD_TESTS=ON" @@ -233,7 +230,6 @@ case "$1" in fbthrift) NAME=fbthrift SRCDIR=cachelib/external/$NAME - update_submodules=yes cmake_custom_params="-DBUILD_SHARED_LIBS=ON" ;; diff --git a/docker/images/build-image.sh b/docker/images/build-image.sh index 985a6e0ff1..1024c8e6d5 100755 --- a/docker/images/build-image.sh +++ b/docker/images/build-image.sh @@ -35,4 +35,4 @@ echo "Build a Docker image tagged with: ${CONTAINER_REG}:${TAG}" docker build -t ${CONTAINER_REG}:${TAG} \ --build-arg http_proxy=$http_proxy \ --build-arg https_proxy=$https_proxy \ - -f ${OS}-${OS_VER}.Dockerfile . + -f ${OS}-${OS_VER}.Dockerfile ../.. # need access to contrib and submodules diff --git a/docker/images/centos-8streams.Dockerfile b/docker/images/centos-8streams.Dockerfile index e0c31226a1..73168e3cb3 100644 --- a/docker/images/centos-8streams.Dockerfile +++ b/docker/images/centos-8streams.Dockerfile @@ -23,8 +23,9 @@ RUN dnf -y install gcc-toolset-12 RUN echo "source /opt/rh/gcc-toolset-12/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] -COPY ./install-cachelib-deps.sh ./install-cachelib-deps.sh -RUN ./install-cachelib-deps.sh +COPY ./contrib ./contrib +COPY ./docker ./docker +COPY ./cachelib/external ./cachelib/external -COPY ./install-dsa-deps.sh ./install-dsa-deps.sh -RUN ./install-dsa-deps.sh +RUN ./docker/images/install-cachelib-deps.sh +RUN ./docker/images/install-dsa-deps.sh diff --git a/docker/images/install-cachelib-deps.sh b/docker/images/install-cachelib-deps.sh index 6d8fbdef7b..b1754a8db5 100755 --- a/docker/images/install-cachelib-deps.sh +++ b/docker/images/install-cachelib-deps.sh @@ -2,13 +2,11 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright 2022, Intel Corporation -git clone -b develop https://github.com/intel/CacheLib CacheLib - -./CacheLib/contrib/prerequisites-centos8.sh +echo 'Defaults env_keep += "HTTPS_PROXY https_proxy HTTP_PROXY http_proxy NO_PROXY no_proxy"' >> /etc/sudoers +./contrib/prerequisites-centos8.sh for pkg in zstd googleflags googlelog googletest sparsemap fmt folly fizz wangle fbthrift ; do - sudo ./CacheLib/contrib/build-package.sh -j -I /opt/ "$pkg" + sudo ./contrib/build-package.sh -j -I /opt/ "$pkg" done -rm -rf CacheLib diff --git a/docker/images/install-dsa-deps.sh b/docker/images/install-dsa-deps.sh index 265011dd70..f3484746b4 100755 --- a/docker/images/install-dsa-deps.sh +++ b/docker/images/install-dsa-deps.sh @@ -15,7 +15,7 @@ rm -rf idxd-config # Install DML Library git clone --recursive https://github.com/intel/DML.git cd DML -git checkout e44443c24d53552b248b9869b1b16f89cd970f52 +git checkout v1.1.0 mkdir build cd build cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo .. From 024e7ade740c372c5c61901b21effc51b20f1caa Mon Sep 17 00:00:00 2001 From: Sounak Gupta Date: Fri, 6 Oct 2023 06:11:35 -0700 Subject: [PATCH 31/31] Baremetal script for running batch expriments with background eviction promotion and DML. Includes json parsing and metric aggregation for analysis and plotting --- baremetal/README.md | 51 + baremetal/accelConfig.sh | 56 + baremetal/aggregateAndFilterTestResults.py | 83 + baremetal/cdn/config.json | 53 + baremetal/cdn/pop.json | 813 + baremetal/cdn/sizes.json | 53634 +++++++++++++++++++ baremetal/parseTestResultIntoCsv.py | 148 + baremetal/runTestAndCovertToJson.py | 201 + baremetal/sample.sh | 127 + 9 files changed, 55166 insertions(+) create mode 100644 baremetal/README.md create mode 100755 baremetal/accelConfig.sh create mode 100755 baremetal/aggregateAndFilterTestResults.py create mode 100644 baremetal/cdn/config.json create mode 100644 baremetal/cdn/pop.json create mode 100644 baremetal/cdn/sizes.json create mode 100755 baremetal/parseTestResultIntoCsv.py create mode 100755 baremetal/runTestAndCovertToJson.py create mode 100755 baremetal/sample.sh diff --git a/baremetal/README.md b/baremetal/README.md new file mode 100644 index 0000000000..08ab6d9328 --- /dev/null +++ b/baremetal/README.md @@ -0,0 +1,51 @@ +# Bare-metal Testing Scripts + + +> ### accelConfig.sh +> +>> Set up DSA devices using accel-config. +>> - OPTIONAL Arg-1: DSA device id. Default: 0 +>> - OPTIONAL Arg-2: Enable/Disable DSA device. Default: yes +>> - OPTIONAL Arg-3: SHARED WQ id. Default: 1 +>> - OPTIONAL Arg-4: ENGINE count. Default: 4 +>> - OUTPUT Verify DSA devices set up is correct using accel-config + + +> ### runTestAndConvertToJson.py +> +>> Run a single test and convert the output TXT to CSV. +>> This script uses numactl to bind all threads to node 0. +>> Run with **sudo** +>> - REQUIRED Arg-1: Cachebench config file path from root directory +>> - REQUIRED Arg-2: DSA device count +>> - REQUIRED Arg-3: Number of background evictors +>> - REQUIRED Arg-4: Eviction batch size +>> - REQUIRED Arg-5: Number of background promoters +>> - REQUIRED Arg-6: Promotion batch size +>> - REQUIRED Arg-7: Output path +>> - OUTPUT txt and json saved in same path + + +> ### aggregateAndFilterTestResults.py +> +>> Gather all output JSON using the file name filter +>> Run with **sudo** +>> - REQUIRED Arg-1: Output json directory path +>> - REQUIRED Arg-2: Filter filenames using this string. Pass null str to gather all files. +>> - OUTPUT Saved in a csv on the output JSON directory path + + +> ### parseTestResultIntoCsv.py +> +>> Parse TXT output and save in JSON format +>> - REQUIRED Arg-1: output txt file name +>> - REQUIRED Arg-2: Tag a string for the text +>> - OUTPUT Save CSV to the same path + + +> ### sample.sh +> +>> This shows how to create a test sequence, then parses results and aggregates it. +>> Run with **sudo** + + diff --git a/baremetal/accelConfig.sh b/baremetal/accelConfig.sh new file mode 100755 index 0000000000..ff0b0b5d89 --- /dev/null +++ b/baremetal/accelConfig.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +echo "OPTIONAL Arg-1: DSA device id. Default: 0" +echo "OPTIONAL Arg-2: Enable/Disable DSA device. Default: yes" +echo "OPTIONAL Arg-3: SHARED WQ id. Default: 1" +echo "OPTIONAL Arg-4: ENGINE count. Default: 4" + +if [ "$#" -ge 5 ]; then + echo "ERROR: Incorrect argument count. Expected arg count <= 4" + exit 1 +fi + +DEVID=${1:-0} +ENABLE=${2:-yes} +SWQID=${3:-1} +NENGS=${4:-4} + +DEV=dsa${DEVID} +SWQ=${DEV}/wq${DEVID}.${SWQID} + +echo "=> ${SWQ}:" +accel-config disable-wq ${SWQ} + +echo "=> ${DEV}:" +accel-config disable-device ${DEV} + +if [ "${ENABLE}" != "yes" ]; then + echo "Exit after disabling ${DEV}." + exit 1 +fi + +for ((i=0; i < ${NENGS}; i++)) +do + echo "=> ${DEV}/engine${DEVID}.${i}" + echo "configured" + accel-config config-engine ${DEV}/engine${DEVID}.${i} --group-id=0 +done + +accel-config config-wq ${SWQ} --group-id=0 +accel-config config-wq ${SWQ} --priority=1 +accel-config config-wq ${SWQ} --wq-size=128 +accel-config config-wq ${SWQ} --max-batch-size=1024 +accel-config config-wq ${SWQ} --max-transfer-size=2147483648 +accel-config config-wq ${SWQ} --block-on-fault=0 +accel-config config-wq ${SWQ} --type=user +accel-config config-wq ${SWQ} --name="dsa-test" +accel-config config-wq ${SWQ} --mode=shared +accel-config config-wq ${SWQ} --threshold=127 +accel-config config-wq ${SWQ} --driver-name="user" + +echo "=> ${DEV}:" +accel-config enable-device ${DEV} + +echo "=> ${SWQ}:" +accel-config enable-wq ${SWQ} + diff --git a/baremetal/aggregateAndFilterTestResults.py b/baremetal/aggregateAndFilterTestResults.py new file mode 100755 index 0000000000..6a33b3f827 --- /dev/null +++ b/baremetal/aggregateAndFilterTestResults.py @@ -0,0 +1,83 @@ +#!/usr/bin/python3 + +import sys +import json +import re +import os +import pandas as pd +import csv + + +def listJson(path, filterStr): + jsonFiles = [jsonFile for jsonFile in os.listdir(path) if jsonFile.endswith('.json') and filterStr in jsonFile] + return [path + f for f in jsonFiles] + + +def getMetrics(files, cbCols, sysCols): + values = {} + for entry in files: + with open(entry, 'r') as jsonFile : + data = json.load(jsonFile) + key = os.path.basename(entry).rsplit(".", 1)[0] + values[key] = {} + for col in cbCols: + values[key][col] = data['cachebench_metrics'][col] + for col in sysCols: + values[key][col] = data['system_metrics'][col] + return values + + +def main(): + args = sys.argv[1:] + if len(args) < 1 or len(args) > 2: + print("Invalid Args. Required : path, filter-string") + exit() + + path = args[0] + filterStr = args[1] if len(args) == 2 else '' + files = listJson(path, filterStr) + + cbCols = [ + 'cache_allocate_api_latency_p90_in_ns', + 'cache_allocate_api_latency_p99_in_ns', + 'cache_find_api_latency_p90_in_ns', + 'cache_find_api_latency_p99_in_ns', + 'cache_background_eviction_latency_p90_in_ns', + 'cache_background_eviction_latency_p99_in_ns', + 'cache_evict_dml_large_item_wait_latency_p90_in_ns', + 'cache_evict_dml_large_item_wait_latency_p99_in_ns', + 'cache_evict_dml_small_item_wait_latency_p90_in_ns', + 'cache_evict_dml_small_item_wait_latency_p99_in_ns', + 'cache_background_promotion_latency_p90_in_ns', + 'cache_background_promotion_latency_p99_in_ns', + 'cache_promote_dml_large_item_wait_latency_p90_in_ns', + 'cache_promote_dml_large_item_wait_latency_p99_in_ns', + 'cache_promote_dml_small_item_wait_latency_p90_in_ns', + 'cache_promote_dml_small_item_wait_latency_p99_in_ns' + ] + + sysCols = [ + 'dsa0/event=0x1,event_category=0x0/', + 'dsa0/event=0x10,event_category=0x1/', + 'dsa0/event=0x2,event_category=0x3/', + 'time_elapsed_in_secs', + 'user_time_seconds', + 'percent_of_cpu_this_job_got' + ] + metrics = getMetrics(files, cbCols, sysCols) + + ''' Save metrics to csv ''' + fields = ['test'] + cbCols + sysCols + csvFile = os.path.join(path , 'metrics.' + filterStr + '.csv') + with open(csvFile, 'w') as f: + w = csv.DictWriter(f, fields) + w.writeheader() + for key, val in sorted(metrics.items()): + row = {'test': key} + row.update(val) + w.writerow(row) + print("Filter: {0} ; Results gathered in {1}".format(filterStr, csvFile)) + + +if __name__ == '__main__': + main() diff --git a/baremetal/cdn/config.json b/baremetal/cdn/config.json new file mode 100644 index 0000000000..a499d660fd --- /dev/null +++ b/baremetal/cdn/config.json @@ -0,0 +1,53 @@ +{ + "cache_config": { + "cacheSizeMB": 32768, + "dsaEnabled": "true", + "minBatchSizeForDsaUsage": 10, + "largeItemMinSize": 4096, + "largeItemBatchEvictDsaUsageFraction": 0.75, + "smallItemBatchEvictDsaUsageFraction": 0.7, + "htBucketPower": 27, + "htBucketLock": 27, + "evictorThreads": 4, + "maxEvictionBatch": 100, + "backgroundEvictorIntervalMilSec": 1, + "promotorThreads": 4, + "maxPromotionBatch": 100, + "backgroundPromoterIntervalMilSec": 1, + "memoryTiers": [ + { + "ratio": 1, + "memBindNodes": 0 + }, + { + "ratio": 1, + "memBindNodes": 1 + } + ], + "poolRebalanceIntervalSec": 0, + "moveOnSlabRelease": false + }, + "test_config": { + "addChainedRatio": 0.0, + "delRatio": 0.0, + "enableLookaside": true, + "getRatio": 0.9911552928593673, + "keySizeRange": [ + 1, + 8, + 64 + ], + "keySizeRangeProbability": [ + 0.3, + 0.7 + ], + "loneGetRatio": 0.008844707140632665, + "numKeys": 8935378, + "numOps": 5000000, + "opRatePerSec": 1000000, + "numThreads": 24, + "popDistFile": "pop.json", + "setRatio": 0.0, + "valSizeDistFile": "sizes.json" + } +} diff --git a/baremetal/cdn/pop.json b/baremetal/cdn/pop.json new file mode 100644 index 0000000000..8f494810e1 --- /dev/null +++ b/baremetal/cdn/pop.json @@ -0,0 +1,813 @@ +{ + "oneHits": 0.008844707140632665, + "popularityWeights": [ + 77039, + 14905, + 13115, + 12714, + 44756, + 9393, + 8211, + 7919, + 7276, + 6964, + 13066, + 6389, + 6073, + 5906, + 5777, + 5612, + 5352, + 10130, + 4915, + 9434, + 4509, + 8878, + 4390, + 4239, + 8082, + 3967, + 7742, + 3827, + 3781, + 3690, + 10953, + 7108, + 6686, + 9831, + 6266, + 6172, + 6046, + 2942, + 8697, + 2865, + 5660, + 5594, + 2756, + 13625, + 2694, + 15912, + 13125, + 2585, + 10236, + 2528, + 15006, + 12295, + 7278, + 2397, + 2345, + 2303, + 4542, + 6735, + 8832, + 8632, + 6384, + 2094, + 6210, + 4080, + 14126, + 1989, + 5904, + 11664, + 3822, + 5634, + 3718, + 3680, + 10908, + 12509, + 10572, + 3470, + 6864, + 6756, + 3344, + 8255, + 8150, + 1603, + 3164, + 1563, + 4605, + 9114, + 7485, + 2964, + 8802, + 5792, + 5720, + 4224, + 1393, + 4137, + 5436, + 2688, + 9303, + 6530, + 3870, + 6370, + 7488, + 9872, + 10989, + 2416, + 4776, + 2346, + 4644, + 3447, + 9096, + 5605, + 5540, + 10960, + 5415, + 6432, + 5305, + 2100, + 4148, + 5130, + 6090, + 2997, + 10868, + 3912, + 6776, + 7632, + 9430, + 6524, + 3684, + 2733, + 4490, + 4445, + 7024, + 4330, + 5142, + 2541, + 7542, + 4140, + 9009, + 4050, + 8000, + 5537, + 7047, + 4650, + 6894, + 11355, + 8940, + 3685, + 10206, + 7909, + 7821, + 5600, + 4830, + 6147, + 2704, + 4683, + 5940, + 5224, + 7740, + 5706, + 5643, + 4952, + 4284, + 9075, + 4784, + 4728, + 8775, + 4053, + 2292, + 9072, + 3927, + 8880, + 8235, + 11403, + 8055, + 4248, + 6300, + 7785, + 7695, + 6591, + 10000, + 4940, + 7335, + 4840, + 5269, + 9480, + 7488, + 6930, + 5941, + 6314, + 7136, + 7938, + 7412, + 11180, + 5950, + 6720, + 8715, + 9840, + 7290, + 8000, + 7505, + 9384, + 4644, + 4596, + 6822, + 6750, + 6660, + 6954, + 5068, + 7160, + 5310, + 9450, + 6920, + 5814, + 6760, + 7682, + 8910, + 4238, + 10626, + 5406, + 7850, + 7750, + 6426, + 8758, + 5662, + 6490, + 9052, + 5491, + 7150, + 7924, + 8400, + 5817, + 5480, + 6775, + 8308, + 3975, + 10742, + 7252, + 5888, + 8602, + 6250, + 7657, + 6588, + 8676, + 6188, + 7285, + 10208, + 7557, + 9266, + 10258, + 7920, + 8029, + 5564, + 10339, + 9776, + 9020, + 8888, + 7363, + 6107, + 6045, + 6948, + 5348, + 7749, + 8228, + 5180, + 6405, + 8869, + 6623, + 8142, + 8925, + 7266, + 8037, + 10647, + 6847, + 7095, + 11084, + 5796, + 9858, + 8321, + 6355, + 9792, + 10721, + 6705, + 9555, + 9860, + 6435, + 11280, + 10286, + 7809, + 13500, + 11837, + 7467, + 12255, + 14097, + 8250, + 12423, + 14762, + 7259, + 13806, + 14720, + 9040, + 15651, + 14715, + 8667, + 15750, + 17819, + 10807, + 12771, + 4018, + 6984, + 10656, + 5130, + 5828, + 12927, + 6532, + 7553, + 14670, + 5340, + 8272, + 16008, + 6708, + 7650, + 18144, + 5644, + 9922, + 20007, + 9520, + 10428, + 23790, + 7700, + 10944, + 26100, + 9324, + 13140, + 24840, + 11573, + 13230, + 26634, + 12308, + 16080, + 26928, + 14235, + 19008, + 30114, + 16678, + 21472, + 35940, + 19470, + 23954, + 36651, + 25088, + 29590, + 43794, + 29468, + 36816, + 51051, + 32700, + 41797, + 61392, + 38305, + 49404, + 73395, + 43780, + 59985, + 87024, + 51168, + 71520, + 107952, + 59128, + 85359, + 136728, + 73360, + 100232, + 165924, + 79808, + 118358, + 207330, + 90712, + 138236, + 264546, + 101452, + 160500, + 347904, + 115345, + 186692, + 456708, + 127120, + 220058, + 635508, + 144415, + 258848, + 910305, + 158620, + 307411, + 1401576, + 177661, + 355160, + 2332701, + 207720, + 432222, + 4664322, + 203395, + 757700, + 16628715, + 210144 + ], + "popularityBuckets": [ + 1, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 3, + 2, + 2, + 3, + 2, + 2, + 2, + 1, + 3, + 1, + 2, + 2, + 1, + 5, + 1, + 6, + 5, + 1, + 4, + 1, + 6, + 5, + 3, + 1, + 1, + 1, + 2, + 3, + 4, + 4, + 3, + 1, + 3, + 2, + 7, + 1, + 3, + 6, + 2, + 3, + 2, + 2, + 6, + 7, + 6, + 2, + 4, + 4, + 2, + 5, + 5, + 1, + 2, + 1, + 3, + 6, + 5, + 2, + 6, + 4, + 4, + 3, + 1, + 3, + 4, + 2, + 7, + 5, + 3, + 5, + 6, + 8, + 9, + 2, + 4, + 2, + 4, + 3, + 8, + 5, + 5, + 10, + 5, + 6, + 5, + 2, + 4, + 5, + 6, + 3, + 11, + 4, + 7, + 8, + 10, + 7, + 4, + 3, + 5, + 5, + 8, + 5, + 6, + 3, + 9, + 5, + 11, + 5, + 10, + 7, + 9, + 6, + 9, + 15, + 12, + 5, + 14, + 11, + 11, + 8, + 7, + 9, + 4, + 7, + 9, + 8, + 12, + 9, + 9, + 8, + 7, + 15, + 8, + 8, + 15, + 7, + 4, + 16, + 7, + 16, + 15, + 21, + 15, + 8, + 12, + 15, + 15, + 13, + 20, + 10, + 15, + 10, + 11, + 20, + 16, + 15, + 13, + 14, + 16, + 18, + 17, + 26, + 14, + 16, + 21, + 24, + 18, + 20, + 19, + 24, + 12, + 12, + 18, + 18, + 18, + 19, + 14, + 20, + 15, + 27, + 20, + 17, + 20, + 23, + 27, + 13, + 33, + 17, + 25, + 25, + 21, + 29, + 19, + 22, + 31, + 19, + 25, + 28, + 30, + 21, + 20, + 25, + 31, + 15, + 41, + 28, + 23, + 34, + 25, + 31, + 27, + 36, + 26, + 31, + 44, + 33, + 41, + 46, + 36, + 37, + 26, + 49, + 47, + 44, + 44, + 37, + 31, + 31, + 36, + 28, + 41, + 44, + 28, + 35, + 49, + 37, + 46, + 51, + 42, + 47, + 63, + 41, + 43, + 68, + 36, + 62, + 53, + 41, + 64, + 71, + 45, + 65, + 68, + 45, + 80, + 74, + 57, + 100, + 89, + 57, + 95, + 111, + 66, + 101, + 122, + 61, + 118, + 128, + 80, + 141, + 135, + 81, + 150, + 173, + 107, + 129, + 41, + 72, + 111, + 54, + 62, + 139, + 71, + 83, + 163, + 60, + 94, + 184, + 78, + 90, + 216, + 68, + 121, + 247, + 119, + 132, + 305, + 100, + 144, + 348, + 126, + 180, + 345, + 163, + 189, + 386, + 181, + 240, + 408, + 219, + 297, + 478, + 269, + 352, + 599, + 330, + 413, + 643, + 448, + 538, + 811, + 556, + 708, + 1001, + 654, + 853, + 1279, + 815, + 1074, + 1631, + 995, + 1395, + 2072, + 1248, + 1788, + 2768, + 1556, + 2307, + 3798, + 2096, + 2948, + 5028, + 2494, + 3818, + 6911, + 3128, + 4937, + 9798, + 3902, + 6420, + 14496, + 5015, + 8486, + 21748, + 6356, + 11582, + 35306, + 8495, + 16178, + 60687, + 11330, + 23647, + 116798, + 16151, + 35516, + 259189, + 25965, + 61746, + 777387, + 40679, + 189425, + 5542905, + 105072 + ] +} diff --git a/baremetal/cdn/sizes.json b/baremetal/cdn/sizes.json new file mode 100644 index 0000000000..fa6dd8ef33 --- /dev/null +++ b/baremetal/cdn/sizes.json @@ -0,0 +1,53634 @@ +{ + "valSizeRangeProbability": [ + 3066, + 983936, + 2, + 245, + 166, + 412, + 43, + 13937, + 2589, + 4590, + 419, + 759, + 384, + 365, + 139, + 245, + 828, + 933, + 260, + 376, + 610, + 208, + 166, + 96, + 646, + 117, + 166, + 187, + 1467, + 596, + 789, + 779, + 446, + 121, + 218, + 266, + 291, + 231, + 314, + 526, + 1085, + 1214, + 1300, + 1696, + 1933, + 2387, + 3168, + 3452, + 3702, + 4792, + 5920, + 6992, + 7359, + 7032, + 7310, + 7581, + 7500, + 7164, + 6974, + 7603, + 8048, + 8230, + 9032, + 9613, + 9283, + 8689, + 8602, + 8505, + 8907, + 9389, + 9744, + 10028, + 9849, + 9775, + 9449, + 9074, + 8965, + 8841, + 8676, + 8520, + 8628, + 8456, + 8422, + 8340, + 8317, + 8523, + 8504, + 8712, + 8669, + 8662, + 8524, + 8151, + 7833, + 7614, + 7119, + 6726, + 6447, + 6235, + 5937, + 5785, + 5663, + 5592, + 5557, + 5568, + 5530, + 5468, + 5542, + 5391, + 5475, + 5533, + 5350, + 5375, + 5389, + 5280, + 5393, + 5287, + 5309, + 5371, + 5370, + 5311, + 5250, + 5346, + 5424, + 5426, + 5174, + 5111, + 5219, + 5244, + 5387, + 5271, + 5343, + 5238, + 4918, + 5102, + 4937, + 5116, + 4977, + 4868, + 4766, + 4732, + 4707, + 4871, + 5093, + 4817, + 4865, + 4732, + 4781, + 4733, + 4718, + 4788, + 4790, + 4683, + 4717, + 4726, + 4704, + 4657, + 4899, + 4853, + 4776, + 4718, + 4916, + 4727, + 4821, + 4788, + 4779, + 4865, + 4798, + 4892, + 4829, + 4743, + 4747, + 4789, + 4677, + 4766, + 4737, + 4673, + 4746, + 4630, + 4595, + 4643, + 4514, + 4442, + 4502, + 4565, + 4501, + 4509, + 4364, + 4535, + 4408, + 4342, + 4458, + 4268, + 4217, + 4202, + 4395, + 4224, + 4219, + 4211, + 4205, + 4151, + 4037, + 4065, + 4141, + 4023, + 4061, + 4037, + 4031, + 4023, + 4040, + 4003, + 4052, + 4152, + 4603, + 5406, + 7031, + 7040, + 5703, + 4931, + 4721, + 4859, + 4478, + 4156, + 4007, + 3975, + 3885, + 3750, + 3823, + 3812, + 3791, + 3770, + 3826, + 3730, + 3700, + 3692, + 3739, + 3611, + 3684, + 3671, + 3581, + 3619, + 3560, + 3651, + 3622, + 3589, + 3591, + 3533, + 3495, + 3492, + 3469, + 3510, + 3469, + 3516, + 3583, + 3455, + 3471, + 3354, + 3416, + 3392, + 3440, + 3442, + 3412, + 3409, + 3463, + 3484, + 3594, + 3812, + 3678, + 3584, + 3558, + 3309, + 3272, + 3327, + 3345, + 3288, + 3307, + 3221, + 3384, + 3398, + 3195, + 3266, + 3116, + 3263, + 3148, + 3234, + 3092, + 3097, + 3068, + 3098, + 3124, + 3067, + 2988, + 3024, + 2985, + 2991, + 2987, + 2954, + 3047, + 3012, + 3058, + 3147, + 2998, + 2980, + 2846, + 2908, + 2935, + 2969, + 2928, + 2903, + 2871, + 2904, + 2895, + 2987, + 2862, + 2878, + 2891, + 2838, + 2866, + 2857, + 2770, + 2847, + 2789, + 2839, + 2850, + 2909, + 2866, + 2805, + 2703, + 2779, + 2726, + 2703, + 2743, + 2741, + 2747, + 2711, + 2790, + 2654, + 2701, + 2787, + 2687, + 2672, + 2727, + 2653, + 2775, + 2627, + 2659, + 2724, + 2628, + 2591, + 2711, + 2720, + 2682, + 2588, + 2614, + 2738, + 2720, + 2684, + 2598, + 2620, + 2714, + 2543, + 2575, + 2581, + 2551, + 2523, + 2563, + 2641, + 2506, + 2669, + 2549, + 2520, + 2585, + 2543, + 2753, + 2659, + 2557, + 2631, + 2645, + 2719, + 2563, + 2501, + 2476, + 2495, + 2487, + 2393, + 2475, + 2481, + 2499, + 2421, + 2464, + 2347, + 2461, + 2396, + 2330, + 2361, + 2424, + 2447, + 2402, + 2370, + 2423, + 2418, + 2372, + 2353, + 2408, + 2352, + 2367, + 2385, + 2346, + 2355, + 2340, + 2346, + 2358, + 2339, + 2349, + 2338, + 2399, + 2375, + 2277, + 2287, + 2450, + 2372, + 2349, + 2382, + 2266, + 2216, + 2384, + 2306, + 2357, + 2288, + 2288, + 2303, + 2280, + 2289, + 2308, + 2348, + 2303, + 2268, + 2288, + 2297, + 2250, + 2208, + 2331, + 2201, + 2375, + 2258, + 2202, + 2321, + 2256, + 2200, + 2178, + 2256, + 2300, + 2181, + 2073, + 2201, + 2194, + 2142, + 2135, + 2179, + 2217, + 2172, + 2195, + 2207, + 2269, + 2191, + 2182, + 2226, + 2234, + 2140, + 2143, + 2129, + 2219, + 2140, + 2147, + 2182, + 2139, + 2083, + 2169, + 2080, + 2071, + 2092, + 2180, + 2090, + 2147, + 2049, + 2059, + 2067, + 2168, + 2156, + 2048, + 2092, + 2130, + 2031, + 2142, + 2215, + 2121, + 2136, + 2046, + 2154, + 2018, + 2138, + 2131, + 2075, + 2076, + 2169, + 2182, + 2091, + 2115, + 2115, + 2109, + 2075, + 2062, + 2060, + 2073, + 1984, + 2143, + 2063, + 2064, + 2066, + 2083, + 2071, + 2090, + 2076, + 2073, + 2098, + 2063, + 2139, + 2221, + 2115, + 2139, + 2208, + 2176, + 2250, + 2247, + 2353, + 2358, + 2329, + 2262, + 2169, + 2233, + 2228, + 2285, + 2064, + 2018, + 2001, + 2005, + 1927, + 1985, + 2053, + 2087, + 2176, + 2316, + 2124, + 2079, + 2012, + 2012, + 1982, + 1989, + 2108, + 2030, + 2041, + 1947, + 2063, + 2077, + 1910, + 1965, + 1989, + 1905, + 1926, + 1948, + 2072, + 1980, + 1973, + 1984, + 1980, + 1957, + 1936, + 1883, + 1939, + 1899, + 1916, + 1912, + 1941, + 1863, + 1938, + 1869, + 1919, + 1898, + 1868, + 1839, + 1924, + 1806, + 1824, + 1862, + 1943, + 1907, + 1789, + 1796, + 1811, + 1884, + 1827, + 1818, + 1889, + 1909, + 1832, + 1823, + 1888, + 1900, + 1770, + 1780, + 1833, + 1831, + 1841, + 1911, + 1886, + 1868, + 1890, + 1807, + 1848, + 1811, + 1740, + 1843, + 1759, + 1809, + 1791, + 1719, + 1745, + 1836, + 1781, + 1831, + 1754, + 1754, + 1692, + 1735, + 1784, + 1764, + 1726, + 1734, + 1761, + 1773, + 1763, + 1747, + 1750, + 1801, + 1711, + 1682, + 1768, + 1750, + 1683, + 1648, + 1671, + 1783, + 1621, + 1740, + 1733, + 1674, + 1767, + 1667, + 1681, + 1680, + 1742, + 1660, + 1698, + 1661, + 1678, + 1666, + 1716, + 1663, + 1728, + 1694, + 1674, + 1679, + 1663, + 1639, + 1728, + 1685, + 1638, + 1689, + 1673, + 1705, + 1667, + 1657, + 1651, + 1682, + 1609, + 1614, + 1681, + 1660, + 1581, + 1674, + 1624, + 1630, + 1641, + 1684, + 1608, + 1640, + 1706, + 1696, + 1626, + 1684, + 1684, + 1629, + 1632, + 1576, + 1576, + 1564, + 1641, + 1576, + 1624, + 1586, + 1602, + 1533, + 1553, + 1536, + 1623, + 1698, + 1582, + 1516, + 1545, + 1563, + 1519, + 1633, + 1521, + 1649, + 1555, + 1632, + 1586, + 1553, + 1567, + 1609, + 1594, + 1592, + 1602, + 1587, + 1561, + 1538, + 1578, + 1534, + 1482, + 1589, + 1486, + 1589, + 1552, + 1513, + 1516, + 1573, + 1543, + 1469, + 1558, + 1513, + 1530, + 1556, + 1605, + 1521, + 1529, + 1530, + 1436, + 1475, + 1539, + 1511, + 1526, + 1540, + 1472, + 1519, + 1503, + 1451, + 1438, + 1500, + 1493, + 1565, + 1521, + 1406, + 1489, + 1562, + 1501, + 1425, + 1445, + 1491, + 1470, + 1448, + 1483, + 1499, + 1535, + 1482, + 1471, + 1475, + 1517, + 1476, + 1496, + 1450, + 1403, + 1442, + 1445, + 1488, + 1443, + 1390, + 1454, + 1471, + 1390, + 1414, + 1434, + 1425, + 1459, + 1414, + 1426, + 1396, + 1433, + 1515, + 1407, + 1477, + 1405, + 1393, + 1506, + 1419, + 1429, + 1360, + 1411, + 1429, + 1361, + 1434, + 1428, + 1440, + 1392, + 1477, + 1380, + 1429, + 1425, + 1423, + 1301, + 1452, + 1371, + 1334, + 1401, + 1420, + 1389, + 1417, + 1390, + 1349, + 1432, + 1446, + 1379, + 1414, + 1350, + 1412, + 1332, + 1353, + 1354, + 1379, + 1383, + 1375, + 1420, + 1348, + 1427, + 1352, + 1343, + 1351, + 1319, + 1413, + 1327, + 1377, + 1363, + 1344, + 1313, + 1345, + 1358, + 1382, + 1397, + 1347, + 1362, + 1316, + 1346, + 1309, + 1407, + 1349, + 1314, + 1281, + 1377, + 1304, + 1388, + 1290, + 1369, + 1358, + 1414, + 1294, + 1356, + 1318, + 1294, + 1346, + 1352, + 1322, + 1307, + 1286, + 1266, + 1303, + 1357, + 1297, + 1338, + 1306, + 1280, + 1358, + 1275, + 1318, + 1260, + 1304, + 1236, + 1327, + 1233, + 1274, + 1298, + 1210, + 1335, + 1288, + 1275, + 1326, + 1351, + 1330, + 1307, + 1265, + 1296, + 1265, + 1323, + 1367, + 1260, + 1250, + 1267, + 1247, + 1185, + 1185, + 1218, + 1279, + 1261, + 1278, + 1259, + 1273, + 1221, + 1268, + 1308, + 1239, + 1218, + 1279, + 1274, + 1241, + 1250, + 1303, + 1217, + 1284, + 1206, + 1241, + 1191, + 1294, + 1255, + 1290, + 1251, + 1224, + 1272, + 1192, + 1212, + 1246, + 1254, + 1255, + 1282, + 1235, + 1265, + 1245, + 1292, + 1244, + 1268, + 1234, + 1231, + 1226, + 1261, + 1266, + 1322, + 1261, + 1277, + 1209, + 1233, + 1209, + 1157, + 1164, + 1229, + 1201, + 1232, + 1276, + 1239, + 1199, + 1205, + 1189, + 1244, + 1200, + 1207, + 1207, + 1208, + 1239, + 1204, + 1246, + 1219, + 1184, + 1240, + 1179, + 1183, + 1133, + 1241, + 1185, + 1176, + 1235, + 1182, + 1127, + 1195, + 1198, + 1192, + 1180, + 1247, + 1164, + 1247, + 1174, + 1096, + 1191, + 1144, + 1181, + 1136, + 1181, + 1183, + 1203, + 1210, + 1151, + 1181, + 1196, + 1211, + 1191, + 1176, + 1186, + 1252, + 1146, + 1198, + 1194, + 1239, + 1132, + 1213, + 1182, + 1179, + 1159, + 1290, + 1187, + 1212, + 1233, + 1192, + 1178, + 1214, + 1235, + 1181, + 1195, + 1118, + 1174, + 1175, + 1158, + 1188, + 1145, + 1238, + 1159, + 1223, + 1109, + 1186, + 1165, + 1119, + 1123, + 1159, + 1150, + 1179, + 1137, + 1166, + 1157, + 1116, + 1117, + 1176, + 1163, + 1110, + 1173, + 1145, + 1121, + 1150, + 1130, + 1169, + 1091, + 1064, + 1107, + 1180, + 1128, + 1100, + 1071, + 1181, + 1072, + 1155, + 1112, + 1129, + 1097, + 1061, + 1173, + 1138, + 1045, + 1101, + 1094, + 1114, + 1130, + 1110, + 1090, + 1152, + 1054, + 1116, + 1121, + 1125, + 1096, + 1092, + 1086, + 1091, + 1099, + 1136, + 1157, + 1166, + 1106, + 1127, + 1057, + 1063, + 1120, + 1131, + 1125, + 1127, + 1075, + 1033, + 1091, + 1037, + 1006, + 1104, + 1129, + 1099, + 1063, + 1061, + 1085, + 1082, + 1075, + 1101, + 1128, + 1094, + 1106, + 1129, + 1129, + 1047, + 1118, + 1083, + 1081, + 1031, + 1106, + 1007, + 1097, + 1095, + 1089, + 1123, + 1036, + 1070, + 1098, + 1059, + 1113, + 1059, + 1114, + 1068, + 1100, + 1026, + 1100, + 1073, + 1050, + 1069, + 1032, + 1069, + 1052, + 1137, + 1113, + 1050, + 1047, + 1027, + 1067, + 1087, + 1032, + 1018, + 1088, + 1035, + 1060, + 1041, + 1096, + 1035, + 1092, + 1040, + 1123, + 1078, + 1097, + 1053, + 1030, + 1033, + 1120, + 1094, + 1050, + 997, + 1056, + 1035, + 1041, + 1028, + 1074, + 1028, + 1036, + 1092, + 1028, + 1033, + 1078, + 1042, + 1084, + 1062, + 1081, + 1016, + 1021, + 1036, + 1017, + 1030, + 1101, + 1030, + 1046, + 1070, + 1028, + 1096, + 996, + 1102, + 1082, + 1039, + 1060, + 1013, + 1010, + 1009, + 1027, + 987, + 1043, + 1102, + 1039, + 1012, + 1036, + 958, + 1020, + 987, + 1019, + 1059, + 1031, + 1007, + 1014, + 977, + 1004, + 997, + 993, + 1005, + 1001, + 1032, + 1015, + 988, + 1014, + 1026, + 996, + 1113, + 1084, + 1005, + 1039, + 990, + 1014, + 995, + 1015, + 1061, + 1032, + 1044, + 1021, + 998, + 1022, + 1016, + 974, + 998, + 1023, + 1030, + 1048, + 1015, + 1035, + 998, + 950, + 1000, + 1017, + 994, + 1022, + 1006, + 989, + 1006, + 988, + 1007, + 1019, + 1064, + 1016, + 1037, + 1044, + 1012, + 1006, + 1011, + 992, + 989, + 1003, + 1002, + 986, + 993, + 1065, + 1027, + 971, + 983, + 1002, + 1006, + 971, + 962, + 1027, + 1000, + 1060, + 986, + 998, + 944, + 957, + 1031, + 1017, + 980, + 1027, + 1002, + 991, + 974, + 978, + 1003, + 981, + 975, + 1005, + 1020, + 993, + 1008, + 944, + 991, + 1025, + 996, + 1028, + 994, + 1017, + 988, + 991, + 939, + 1003, + 952, + 973, + 1026, + 972, + 961, + 984, + 988, + 952, + 975, + 998, + 939, + 973, + 978, + 981, + 960, + 969, + 989, + 954, + 977, + 957, + 1000, + 963, + 986, + 936, + 953, + 971, + 994, + 973, + 935, + 965, + 999, + 1014, + 970, + 994, + 970, + 935, + 906, + 921, + 926, + 984, + 944, + 966, + 1019, + 916, + 963, + 1000, + 941, + 948, + 934, + 916, + 898, + 923, + 981, + 968, + 942, + 963, + 921, + 1009, + 933, + 979, + 1021, + 989, + 1008, + 971, + 994, + 984, + 968, + 988, + 965, + 958, + 970, + 937, + 996, + 970, + 933, + 987, + 944, + 934, + 984, + 921, + 971, + 921, + 928, + 961, + 940, + 991, + 979, + 964, + 886, + 965, + 966, + 891, + 942, + 929, + 994, + 945, + 964, + 916, + 970, + 874, + 942, + 898, + 925, + 920, + 981, + 921, + 964, + 916, + 936, + 900, + 983, + 922, + 920, + 929, + 886, + 922, + 896, + 921, + 946, + 935, + 948, + 940, + 925, + 933, + 913, + 899, + 920, + 950, + 982, + 933, + 872, + 997, + 890, + 966, + 925, + 909, + 894, + 918, + 936, + 950, + 906, + 948, + 964, + 933, + 935, + 951, + 907, + 931, + 927, + 955, + 881, + 951, + 948, + 892, + 935, + 933, + 929, + 920, + 901, + 931, + 888, + 996, + 905, + 909, + 884, + 876, + 914, + 976, + 964, + 875, + 948, + 908, + 884, + 938, + 863, + 874, + 938, + 894, + 936, + 896, + 898, + 927, + 949, + 892, + 923, + 931, + 925, + 901, + 896, + 920, + 874, + 895, + 898, + 898, + 954, + 864, + 946, + 916, + 869, + 909, + 929, + 932, + 945, + 879, + 934, + 898, + 843, + 929, + 916, + 892, + 842, + 878, + 861, + 905, + 894, + 906, + 897, + 871, + 907, + 921, + 918, + 845, + 872, + 871, + 862, + 832, + 929, + 905, + 985, + 859, + 920, + 925, + 920, + 881, + 879, + 890, + 880, + 871, + 881, + 912, + 896, + 912, + 896, + 912, + 857, + 903, + 870, + 901, + 898, + 912, + 907, + 884, + 878, + 921, + 914, + 915, + 878, + 869, + 850, + 856, + 914, + 873, + 909, + 892, + 899, + 818, + 876, + 885, + 924, + 885, + 922, + 856, + 838, + 875, + 877, + 905, + 858, + 913, + 886, + 855, + 892, + 933, + 866, + 859, + 874, + 902, + 860, + 887, + 887, + 917, + 853, + 857, + 943, + 830, + 897, + 881, + 893, + 919, + 848, + 922, + 873, + 803, + 847, + 867, + 855, + 836, + 832, + 888, + 887, + 869, + 878, + 902, + 871, + 880, + 893, + 870, + 885, + 881, + 879, + 890, + 876, + 885, + 851, + 862, + 843, + 810, + 895, + 865, + 857, + 837, + 862, + 825, + 874, + 918, + 841, + 877, + 793, + 809, + 883, + 872, + 831, + 832, + 817, + 842, + 856, + 798, + 874, + 863, + 789, + 872, + 836, + 841, + 877, + 909, + 891, + 803, + 848, + 857, + 879, + 878, + 897, + 828, + 875, + 852, + 849, + 863, + 834, + 818, + 884, + 861, + 840, + 845, + 889, + 852, + 859, + 852, + 847, + 828, + 832, + 895, + 867, + 855, + 829, + 853, + 896, + 787, + 864, + 878, + 783, + 821, + 805, + 826, + 872, + 832, + 842, + 856, + 826, + 823, + 820, + 818, + 784, + 859, + 800, + 840, + 831, + 793, + 878, + 854, + 823, + 792, + 882, + 826, + 867, + 825, + 799, + 842, + 843, + 829, + 900, + 824, + 837, + 788, + 860, + 820, + 811, + 828, + 796, + 818, + 902, + 805, + 787, + 859, + 865, + 847, + 870, + 786, + 840, + 865, + 798, + 839, + 844, + 828, + 858, + 857, + 865, + 806, + 819, + 781, + 865, + 820, + 806, + 834, + 831, + 771, + 761, + 805, + 903, + 840, + 828, + 855, + 832, + 800, + 808, + 849, + 857, + 809, + 843, + 865, + 810, + 847, + 806, + 790, + 850, + 836, + 839, + 826, + 808, + 793, + 821, + 858, + 858, + 799, + 791, + 864, + 818, + 781, + 832, + 820, + 816, + 782, + 808, + 813, + 777, + 824, + 861, + 815, + 795, + 811, + 858, + 793, + 754, + 848, + 845, + 805, + 792, + 741, + 781, + 800, + 782, + 792, + 792, + 831, + 771, + 803, + 846, + 840, + 809, + 824, + 802, + 777, + 795, + 749, + 753, + 828, + 817, + 821, + 847, + 789, + 831, + 864, + 843, + 780, + 755, + 789, + 794, + 735, + 823, + 806, + 787, + 825, + 828, + 813, + 779, + 796, + 799, + 771, + 782, + 771, + 799, + 837, + 800, + 826, + 801, + 840, + 842, + 773, + 738, + 791, + 802, + 817, + 796, + 810, + 788, + 797, + 766, + 782, + 747, + 787, + 776, + 775, + 797, + 783, + 770, + 810, + 824, + 778, + 776, + 779, + 773, + 797, + 769, + 780, + 755, + 841, + 841, + 802, + 728, + 793, + 824, + 787, + 758, + 729, + 811, + 756, + 827, + 814, + 820, + 734, + 817, + 788, + 813, + 805, + 763, + 790, + 763, + 759, + 782, + 726, + 795, + 833, + 782, + 833, + 755, + 834, + 767, + 808, + 818, + 783, + 809, + 752, + 775, + 781, + 841, + 772, + 798, + 727, + 787, + 834, + 750, + 761, + 859, + 744, + 735, + 782, + 778, + 775, + 732, + 746, + 798, + 716, + 762, + 798, + 753, + 753, + 759, + 767, + 747, + 809, + 757, + 736, + 775, + 770, + 761, + 742, + 746, + 744, + 779, + 774, + 757, + 784, + 795, + 754, + 831, + 779, + 771, + 702, + 761, + 751, + 710, + 779, + 760, + 733, + 784, + 789, + 747, + 757, + 755, + 778, + 736, + 805, + 774, + 759, + 736, + 727, + 775, + 717, + 726, + 783, + 766, + 746, + 770, + 762, + 720, + 725, + 328, + 352, + 370, + 355, + 377, + 331, + 333, + 343, + 355, + 377, + 355, + 350, + 326, + 337, + 342, + 363, + 340, + 355, + 317, + 365, + 362, + 337, + 366, + 312, + 315, + 355, + 376, + 324, + 318, + 323, + 335, + 305, + 365, + 343, + 348, + 352, + 352, + 364, + 335, + 374, + 346, + 345, + 376, + 346, + 341, + 330, + 320, + 336, + 323, + 338, + 325, + 343, + 292, + 351, + 328, + 294, + 330, + 338, + 320, + 360, + 291, + 300, + 329, + 346, + 307, + 316, + 337, + 351, + 333, + 343, + 348, + 357, + 359, + 336, + 356, + 336, + 324, + 323, + 311, + 321, + 357, + 341, + 317, + 350, + 349, + 337, + 311, + 322, + 352, + 376, + 326, + 327, + 299, + 368, + 338, + 350, + 340, + 316, + 338, + 332, + 344, + 322, + 359, + 345, + 336, + 331, + 356, + 311, + 306, + 368, + 344, + 369, + 326, + 321, + 297, + 338, + 337, + 337, + 321, + 329, + 328, + 322, + 292, + 349, + 318, + 355, + 322, + 340, + 337, + 319, + 339, + 328, + 366, + 330, + 322, + 311, + 328, + 336, + 335, + 326, + 319, + 341, + 335, + 343, + 307, + 333, + 311, + 310, + 314, + 368, + 343, + 312, + 324, + 326, + 312, + 357, + 328, + 348, + 336, + 339, + 299, + 287, + 305, + 298, + 275, + 292, + 329, + 326, + 306, + 287, + 324, + 329, + 342, + 321, + 321, + 300, + 315, + 316, + 324, + 281, + 361, + 318, + 338, + 297, + 305, + 334, + 366, + 321, + 327, + 313, + 309, + 326, + 325, + 351, + 332, + 335, + 331, + 342, + 305, + 340, + 316, + 345, + 317, + 320, + 322, + 314, + 306, + 301, + 312, + 328, + 328, + 341, + 311, + 305, + 275, + 289, + 338, + 335, + 345, + 322, + 340, + 297, + 315, + 346, + 346, + 325, + 344, + 324, + 285, + 303, + 283, + 311, + 298, + 337, + 335, + 330, + 316, + 320, + 296, + 320, + 329, + 305, + 293, + 315, + 325, + 301, + 269, + 277, + 299, + 320, + 334, + 301, + 295, + 334, + 326, + 302, + 332, + 311, + 307, + 276, + 352, + 290, + 299, + 331, + 299, + 297, + 308, + 302, + 291, + 294, + 323, + 327, + 323, + 326, + 271, + 290, + 291, + 291, + 337, + 292, + 299, + 283, + 309, + 337, + 301, + 299, + 303, + 286, + 300, + 335, + 305, + 285, + 316, + 293, + 318, + 310, + 294, + 309, + 302, + 300, + 347, + 311, + 297, + 311, + 308, + 318, + 325, + 293, + 258, + 320, + 333, + 308, + 323, + 286, + 322, + 334, + 297, + 284, + 283, + 316, + 288, + 311, + 287, + 293, + 303, + 292, + 275, + 293, + 294, + 318, + 314, + 306, + 293, + 278, + 293, + 289, + 287, + 295, + 285, + 316, + 309, + 274, + 302, + 301, + 323, + 289, + 288, + 296, + 302, + 337, + 266, + 315, + 302, + 325, + 279, + 314, + 293, + 281, + 306, + 288, + 314, + 311, + 275, + 294, + 293, + 289, + 352, + 273, + 308, + 289, + 283, + 304, + 319, + 314, + 302, + 287, + 298, + 265, + 303, + 294, + 287, + 299, + 292, + 301, + 302, + 287, + 306, + 282, + 300, + 277, + 311, + 319, + 313, + 286, + 293, + 297, + 310, + 265, + 263, + 267, + 307, + 305, + 276, + 271, + 293, + 274, + 321, + 288, + 293, + 258, + 312, + 271, + 269, + 316, + 302, + 288, + 289, + 319, + 276, + 297, + 313, + 285, + 268, + 313, + 270, + 292, + 257, + 269, + 273, + 299, + 310, + 298, + 274, + 270, + 308, + 281, + 296, + 305, + 272, + 286, + 273, + 316, + 284, + 291, + 274, + 292, + 289, + 295, + 297, + 298, + 304, + 254, + 330, + 274, + 301, + 278, + 264, + 249, + 289, + 300, + 319, + 322, + 304, + 261, + 250, + 291, + 302, + 283, + 254, + 292, + 314, + 273, + 273, + 279, + 277, + 276, + 248, + 295, + 254, + 288, + 271, + 286, + 255, + 273, + 254, + 285, + 293, + 289, + 283, + 313, + 284, + 273, + 255, + 289, + 278, + 250, + 251, + 253, + 270, + 296, + 278, + 256, + 274, + 249, + 286, + 299, + 285, + 268, + 269, + 261, + 267, + 275, + 246, + 281, + 255, + 258, + 307, + 255, + 269, + 284, + 279, + 288, + 248, + 260, + 277, + 274, + 300, + 278, + 279, + 266, + 258, + 277, + 260, + 287, + 306, + 240, + 256, + 279, + 293, + 254, + 250, + 275, + 252, + 267, + 261, + 268, + 257, + 267, + 295, + 273, + 287, + 254, + 262, + 264, + 256, + 248, + 233, + 256, + 267, + 276, + 300, + 251, + 273, + 263, + 260, + 265, + 264, + 247, + 264, + 275, + 268, + 245, + 269, + 234, + 241, + 237, + 262, + 289, + 268, + 256, + 265, + 232, + 257, + 286, + 254, + 270, + 249, + 251, + 241, + 243, + 262, + 238, + 234, + 273, + 255, + 286, + 275, + 254, + 259, + 253, + 241, + 287, + 259, + 242, + 260, + 274, + 259, + 250, + 292, + 264, + 254, + 277, + 268, + 245, + 250, + 237, + 289, + 261, + 246, + 261, + 262, + 255, + 285, + 274, + 266, + 258, + 262, + 229, + 239, + 241, + 249, + 270, + 253, + 274, + 266, + 241, + 255, + 251, + 256, + 249, + 261, + 246, + 252, + 234, + 222, + 236, + 231, + 294, + 229, + 256, + 254, + 273, + 256, + 279, + 243, + 291, + 233, + 267, + 274, + 256, + 236, + 237, + 243, + 274, + 258, + 212, + 243, + 231, + 210, + 261, + 249, + 234, + 241, + 266, + 236, + 236, + 255, + 255, + 238, + 244, + 238, + 220, + 244, + 226, + 284, + 253, + 248, + 222, + 250, + 246, + 230, + 225, + 228, + 249, + 242, + 214, + 243, + 263, + 237, + 250, + 240, + 217, + 273, + 235, + 237, + 249, + 263, + 238, + 252, + 242, + 237, + 260, + 216, + 247, + 213, + 273, + 223, + 248, + 260, + 230, + 213, + 276, + 223, + 266, + 239, + 249, + 234, + 222, + 240, + 225, + 252, + 241, + 259, + 262, + 228, + 256, + 261, + 196, + 239, + 260, + 246, + 256, + 217, + 264, + 226, + 209, + 229, + 237, + 250, + 237, + 234, + 227, + 210, + 241, + 247, + 256, + 255, + 230, + 223, + 265, + 242, + 230, + 211, + 237, + 224, + 247, + 247, + 236, + 223, + 259, + 256, + 255, + 202, + 236, + 211, + 216, + 236, + 253, + 239, + 217, + 220, + 262, + 230, + 233, + 210, + 202, + 254, + 238, + 227, + 245, + 235, + 247, + 208, + 233, + 236, + 228, + 251, + 214, + 256, + 236, + 227, + 246, + 246, + 221, + 247, + 238, + 256, + 234, + 186, + 242, + 249, + 228, + 186, + 247, + 218, + 217, + 229, + 213, + 247, + 220, + 235, + 216, + 215, + 215, + 215, + 217, + 229, + 205, + 248, + 219, + 239, + 235, + 231, + 252, + 238, + 182, + 215, + 239, + 240, + 209, + 204, + 225, + 203, + 231, + 219, + 228, + 212, + 243, + 223, + 213, + 234, + 207, + 198, + 228, + 236, + 226, + 215, + 212, + 232, + 210, + 223, + 229, + 227, + 193, + 245, + 185, + 183, + 232, + 222, + 197, + 234, + 191, + 230, + 216, + 205, + 225, + 233, + 212, + 211, + 221, + 196, + 231, + 215, + 221, + 197, + 221, + 204, + 244, + 221, + 231, + 197, + 227, + 232, + 213, + 218, + 209, + 220, + 226, + 236, + 203, + 227, + 222, + 234, + 212, + 212, + 220, + 217, + 195, + 197, + 208, + 211, + 225, + 233, + 204, + 245, + 229, + 204, + 240, + 209, + 194, + 182, + 191, + 199, + 229, + 198, + 215, + 205, + 173, + 177, + 224, + 203, + 212, + 195, + 196, + 198, + 217, + 216, + 202, + 203, + 192, + 218, + 214, + 224, + 209, + 206, + 181, + 227, + 203, + 188, + 198, + 219, + 201, + 212, + 218, + 208, + 211, + 202, + 232, + 177, + 190, + 213, + 201, + 221, + 229, + 218, + 218, + 203, + 232, + 209, + 190, + 225, + 199, + 198, + 204, + 214, + 222, + 180, + 204, + 196, + 180, + 198, + 196, + 187, + 210, + 199, + 178, + 215, + 214, + 229, + 201, + 208, + 209, + 181, + 201, + 207, + 231, + 205, + 205, + 179, + 207, + 206, + 209, + 179, + 206, + 177, + 204, + 227, + 207, + 172, + 193, + 191, + 191, + 224, + 188, + 221, + 184, + 205, + 169, + 189, + 199, + 200, + 217, + 199, + 179, + 186, + 196, + 197, + 179, + 234, + 159, + 208, + 208, + 201, + 222, + 193, + 189, + 208, + 179, + 216, + 195, + 201, + 190, + 222, + 206, + 187, + 234, + 206, + 216, + 187, + 192, + 215, + 194, + 190, + 193, + 205, + 198, + 196, + 192, + 203, + 185, + 191, + 210, + 197, + 185, + 183, + 185, + 208, + 195, + 167, + 199, + 176, + 184, + 194, + 194, + 184, + 220, + 190, + 185, + 199, + 175, + 199, + 169, + 158, + 177, + 216, + 196, + 196, + 189, + 197, + 199, + 222, + 189, + 191, + 202, + 193, + 186, + 194, + 211, + 172, + 187, + 186, + 182, + 171, + 197, + 204, + 215, + 228, + 176, + 177, + 188, + 199, + 201, + 188, + 198, + 179, + 195, + 162, + 204, + 181, + 188, + 163, + 184, + 202, + 191, + 207, + 192, + 158, + 182, + 199, + 196, + 175, + 190, + 176, + 160, + 204, + 209, + 179, + 191, + 190, + 174, + 191, + 169, + 184, + 190, + 165, + 183, + 175, + 199, + 202, + 172, + 191, + 168, + 208, + 197, + 175, + 190, + 178, + 180, + 163, + 192, + 170, + 154, + 204, + 177, + 187, + 144, + 167, + 188, + 176, + 147, + 166, + 191, + 188, + 162, + 205, + 172, + 208, + 187, + 194, + 182, + 196, + 180, + 142, + 186, + 161, + 164, + 179, + 191, + 161, + 177, + 182, + 200, + 161, + 184, + 178, + 166, + 196, + 174, + 184, + 189, + 187, + 189, + 205, + 159, + 167, + 151, + 172, + 156, + 178, + 182, + 177, + 177, + 183, + 161, + 156, + 163, + 190, + 167, + 173, + 181, + 196, + 160, + 196, + 153, + 159, + 163, + 189, + 193, + 185, + 173, + 173, + 186, + 178, + 166, + 170, + 176, + 174, + 178, + 166, + 182, + 172, + 173, + 196, + 162, + 165, + 173, + 168, + 167, + 167, + 180, + 168, + 179, + 180, + 168, + 176, + 163, + 167, + 207, + 167, + 182, + 168, + 172, + 178, + 155, + 152, + 170, + 170, + 149, + 172, + 163, + 187, + 174, + 186, + 174, + 153, + 152, + 149, + 148, + 163, + 167, + 167, + 166, + 152, + 182, + 174, + 148, + 139, + 166, + 169, + 167, + 148, + 148, + 158, + 168, + 158, + 156, + 146, + 161, + 148, + 167, + 178, + 165, + 164, + 163, + 147, + 162, + 175, + 152, + 157, + 181, + 142, + 176, + 146, + 178, + 155, + 162, + 160, + 166, + 169, + 147, + 168, + 158, + 162, + 160, + 173, + 152, + 147, + 132, + 179, + 165, + 142, + 168, + 158, + 168, + 164, + 144, + 171, + 156, + 167, + 154, + 155, + 166, + 164, + 171, + 163, + 150, + 155, + 158, + 172, + 163, + 170, + 154, + 181, + 140, + 162, + 160, + 146, + 145, + 158, + 159, + 153, + 162, + 175, + 159, + 162, + 178, + 152, + 158, + 150, + 148, + 153, + 173, + 143, + 153, + 158, + 149, + 174, + 146, + 155, + 129, + 141, + 162, + 151, + 157, + 156, + 155, + 142, + 131, + 152, + 157, + 146, + 158, + 145, + 161, + 148, + 144, + 146, + 171, + 161, + 155, + 168, + 151, + 119, + 130, + 174, + 152, + 165, + 136, + 162, + 148, + 161, + 162, + 143, + 180, + 138, + 182, + 152, + 171, + 144, + 139, + 146, + 154, + 148, + 143, + 169, + 162, + 148, + 170, + 153, + 149, + 146, + 151, + 137, + 156, + 154, + 136, + 157, + 143, + 141, + 176, + 159, + 138, + 151, + 146, + 141, + 170, + 163, + 142, + 163, + 142, + 139, + 152, + 150, + 152, + 161, + 142, + 182, + 145, + 150, + 160, + 145, + 150, + 138, + 158, + 178, + 134, + 143, + 147, + 136, + 161, + 134, + 143, + 154, + 126, + 139, + 155, + 136, + 139, + 153, + 131, + 140, + 144, + 139, + 162, + 152, + 154, + 138, + 127, + 137, + 134, + 145, + 151, + 154, + 137, + 156, + 129, + 149, + 147, + 149, + 142, + 139, + 130, + 155, + 149, + 157, + 138, + 146, + 145, + 145, + 151, + 160, + 139, + 130, + 158, + 141, + 134, + 140, + 154, + 131, + 140, + 141, + 153, + 129, + 148, + 156, + 146, + 152, + 133, + 154, + 150, + 135, + 143, + 159, + 162, + 142, + 127, + 129, + 148, + 134, + 129, + 146, + 132, + 126, + 159, + 120, + 141, + 142, + 138, + 127, + 132, + 152, + 139, + 152, + 144, + 136, + 144, + 160, + 151, + 123, + 145, + 154, + 143, + 131, + 143, + 127, + 155, + 141, + 129, + 132, + 128, + 123, + 158, + 144, + 116, + 148, + 118, + 103, + 141, + 129, + 129, + 130, + 124, + 134, + 143, + 149, + 132, + 121, + 151, + 154, + 139, + 135, + 123, + 134, + 128, + 116, + 151, + 153, + 144, + 146, + 123, + 136, + 127, + 144, + 144, + 141, + 130, + 142, + 161, + 112, + 133, + 141, + 114, + 129, + 144, + 135, + 153, + 145, + 132, + 136, + 135, + 122, + 128, + 120, + 144, + 147, + 153, + 122, + 148, + 121, + 142, + 106, + 126, + 148, + 116, + 139, + 126, + 162, + 123, + 135, + 126, + 137, + 127, + 129, + 129, + 136, + 136, + 147, + 147, + 140, + 132, + 135, + 134, + 127, + 133, + 120, + 119, + 141, + 114, + 141, + 127, + 130, + 129, + 136, + 120, + 144, + 110, + 146, + 135, + 131, + 115, + 131, + 118, + 120, + 129, + 127, + 116, + 121, + 144, + 142, + 132, + 134, + 119, + 126, + 115, + 141, + 122, + 119, + 118, + 138, + 126, + 135, + 128, + 149, + 136, + 105, + 121, + 135, + 123, + 125, + 131, + 137, + 116, + 145, + 131, + 129, + 114, + 125, + 123, + 135, + 116, + 134, + 121, + 116, + 103, + 109, + 122, + 144, + 120, + 128, + 125, + 121, + 114, + 100, + 135, + 138, + 129, + 119, + 143, + 97, + 108, + 127, + 137, + 124, + 115, + 124, + 124, + 137, + 109, + 137, + 109, + 123, + 118, + 138, + 111, + 123, + 122, + 112, + 123, + 106, + 115, + 129, + 116, + 120, + 128, + 105, + 117, + 130, + 115, + 132, + 125, + 115, + 125, + 104, + 120, + 125, + 119, + 132, + 109, + 120, + 111, + 146, + 113, + 120, + 104, + 128, + 138, + 139, + 120, + 112, + 138, + 115, + 130, + 120, + 106, + 121, + 105, + 117, + 115, + 114, + 104, + 134, + 104, + 141, + 119, + 116, + 116, + 122, + 131, + 129, + 119, + 128, + 102, + 112, + 130, + 109, + 107, + 121, + 105, + 107, + 130, + 116, + 108, + 134, + 115, + 116, + 135, + 120, + 125, + 107, + 115, + 114, + 118, + 116, + 117, + 101, + 120, + 105, + 126, + 101, + 108, + 114, + 104, + 129, + 131, + 110, + 136, + 112, + 131, + 120, + 123, + 117, + 111, + 125, + 124, + 123, + 108, + 106, + 115, + 134, + 116, + 123, + 133, + 101, + 132, + 108, + 115, + 122, + 114, + 116, + 118, + 108, + 107, + 121, + 124, + 108, + 125, + 119, + 106, + 118, + 130, + 111, + 100, + 111, + 102, + 105, + 106, + 113, + 113, + 117, + 113, + 86, + 105, + 117, + 129, + 118, + 112, + 117, + 134, + 89, + 99, + 126, + 105, + 104, + 99, + 134, + 129, + 117, + 103, + 104, + 125, + 101, + 115, + 100, + 105, + 102, + 99, + 108, + 116, + 96, + 125, + 107, + 110, + 111, + 106, + 125, + 122, + 103, + 96, + 105, + 96, + 130, + 97, + 104, + 127, + 114, + 116, + 101, + 85, + 119, + 100, + 113, + 116, + 102, + 108, + 114, + 118, + 118, + 113, + 128, + 118, + 120, + 72, + 103, + 106, + 97, + 103, + 118, + 103, + 108, + 117, + 96, + 101, + 108, + 103, + 105, + 127, + 92, + 113, + 87, + 98, + 112, + 121, + 109, + 114, + 98, + 99, + 104, + 117, + 93, + 91, + 109, + 93, + 126, + 109, + 108, + 119, + 108, + 104, + 95, + 103, + 119, + 111, + 94, + 102, + 108, + 110, + 102, + 116, + 92, + 105, + 99, + 108, + 113, + 107, + 111, + 97, + 116, + 97, + 111, + 82, + 95, + 99, + 116, + 105, + 97, + 113, + 90, + 100, + 99, + 105, + 93, + 109, + 105, + 118, + 113, + 119, + 108, + 122, + 106, + 103, + 105, + 107, + 91, + 112, + 107, + 100, + 105, + 120, + 107, + 108, + 101, + 111, + 104, + 118, + 129, + 99, + 99, + 103, + 88, + 97, + 87, + 100, + 101, + 101, + 109, + 82, + 90, + 72, + 82, + 126, + 94, + 108, + 88, + 83, + 106, + 97, + 92, + 111, + 90, + 112, + 93, + 131, + 99, + 106, + 118, + 97, + 81, + 116, + 94, + 116, + 106, + 96, + 92, + 110, + 103, + 105, + 108, + 85, + 111, + 93, + 94, + 112, + 96, + 101, + 97, + 117, + 85, + 111, + 82, + 104, + 97, + 81, + 95, + 85, + 95, + 108, + 92, + 104, + 119, + 103, + 103, + 98, + 121, + 105, + 93, + 90, + 107, + 86, + 95, + 98, + 90, + 88, + 104, + 81, + 88, + 109, + 132, + 96, + 89, + 83, + 97, + 92, + 101, + 95, + 97, + 92, + 102, + 92, + 83, + 116, + 76, + 97, + 96, + 97, + 107, + 116, + 84, + 97, + 98, + 105, + 109, + 106, + 101, + 83, + 94, + 110, + 102, + 101, + 101, + 96, + 103, + 115, + 90, + 99, + 108, + 91, + 110, + 101, + 96, + 91, + 90, + 97, + 108, + 111, + 87, + 81, + 94, + 104, + 102, + 112, + 80, + 112, + 88, + 76, + 116, + 104, + 82, + 84, + 95, + 82, + 101, + 96, + 93, + 98, + 85, + 89, + 91, + 87, + 79, + 103, + 94, + 83, + 95, + 103, + 83, + 98, + 89, + 100, + 91, + 94, + 86, + 100, + 94, + 104, + 88, + 91, + 95, + 89, + 104, + 99, + 101, + 82, + 104, + 82, + 99, + 95, + 114, + 101, + 95, + 77, + 93, + 87, + 96, + 98, + 91, + 103, + 80, + 91, + 96, + 92, + 88, + 75, + 101, + 102, + 100, + 108, + 90, + 82, + 86, + 84, + 84, + 84, + 91, + 75, + 85, + 118, + 83, + 100, + 73, + 79, + 88, + 94, + 92, + 87, + 90, + 79, + 107, + 106, + 101, + 86, + 83, + 85, + 90, + 87, + 80, + 87, + 98, + 89, + 80, + 80, + 78, + 82, + 85, + 101, + 109, + 82, + 84, + 86, + 72, + 92, + 96, + 105, + 87, + 89, + 75, + 75, + 81, + 86, + 103, + 90, + 105, + 89, + 102, + 84, + 98, + 82, + 75, + 78, + 82, + 85, + 104, + 87, + 91, + 100, + 89, + 99, + 100, + 90, + 91, + 96, + 76, + 84, + 88, + 88, + 88, + 91, + 79, + 68, + 94, + 85, + 88, + 79, + 73, + 74, + 80, + 82, + 91, + 86, + 75, + 95, + 101, + 85, + 84, + 83, + 94, + 99, + 78, + 85, + 84, + 106, + 88, + 76, + 81, + 82, + 81, + 87, + 78, + 72, + 95, + 85, + 87, + 92, + 79, + 97, + 108, + 75, + 83, + 93, + 79, + 86, + 85, + 88, + 77, + 93, + 72, + 79, + 75, + 77, + 72, + 69, + 87, + 93, + 80, + 80, + 80, + 74, + 72, + 86, + 80, + 85, + 80, + 93, + 96, + 77, + 85, + 74, + 73, + 80, + 91, + 88, + 81, + 76, + 82, + 71, + 98, + 76, + 71, + 72, + 72, + 82, + 67, + 74, + 90, + 80, + 94, + 87, + 84, + 72, + 93, + 100, + 91, + 79, + 98, + 89, + 88, + 83, + 79, + 73, + 92, + 92, + 85, + 83, + 66, + 74, + 83, + 73, + 68, + 77, + 86, + 76, + 75, + 72, + 78, + 78, + 72, + 78, + 79, + 80, + 85, + 85, + 76, + 69, + 86, + 80, + 87, + 92, + 80, + 79, + 69, + 81, + 81, + 73, + 79, + 93, + 77, + 84, + 74, + 94, + 89, + 65, + 81, + 74, + 80, + 65, + 73, + 79, + 96, + 86, + 85, + 68, + 74, + 80, + 83, + 69, + 105, + 72, + 63, + 98, + 81, + 84, + 59, + 97, + 75, + 74, + 85, + 90, + 91, + 73, + 65, + 77, + 66, + 77, + 81, + 75, + 67, + 78, + 102, + 75, + 77, + 81, + 89, + 72, + 92, + 81, + 83, + 80, + 83, + 77, + 72, + 65, + 79, + 73, + 68, + 81, + 79, + 92, + 54, + 82, + 75, + 86, + 67, + 67, + 81, + 68, + 72, + 67, + 69, + 65, + 78, + 74, + 65, + 88, + 56, + 77, + 81, + 77, + 68, + 73, + 74, + 85, + 69, + 82, + 71, + 65, + 79, + 70, + 61, + 89, + 70, + 75, + 70, + 72, + 84, + 81, + 81, + 81, + 67, + 71, + 70, + 68, + 77, + 54, + 71, + 69, + 82, + 69, + 77, + 86, + 66, + 76, + 68, + 87, + 67, + 85, + 79, + 80, + 67, + 77, + 81, + 64, + 77, + 88, + 88, + 75, + 81, + 70, + 78, + 64, + 74, + 74, + 69, + 70, + 76, + 83, + 78, + 81, + 76, + 85, + 73, + 64, + 71, + 61, + 80, + 73, + 77, + 88, + 61, + 73, + 68, + 71, + 72, + 72, + 81, + 73, + 58, + 67, + 64, + 63, + 73, + 63, + 74, + 66, + 75, + 66, + 77, + 74, + 53, + 68, + 73, + 83, + 68, + 77, + 75, + 66, + 62, + 70, + 82, + 64, + 75, + 73, + 76, + 71, + 84, + 68, + 85, + 80, + 69, + 63, + 64, + 73, + 74, + 78, + 63, + 66, + 70, + 66, + 68, + 77, + 64, + 59, + 76, + 75, + 72, + 73, + 80, + 57, + 72, + 66, + 56, + 74, + 61, + 58, + 64, + 60, + 66, + 76, + 63, + 61, + 54, + 65, + 58, + 65, + 67, + 87, + 66, + 84, + 62, + 59, + 90, + 65, + 76, + 68, + 64, + 71, + 64, + 69, + 58, + 59, + 91, + 56, + 64, + 72, + 65, + 67, + 78, + 82, + 77, + 60, + 85, + 65, + 58, + 63, + 57, + 76, + 89, + 76, + 57, + 77, + 67, + 73, + 77, + 70, + 58, + 59, + 73, + 72, + 55, + 61, + 83, + 61, + 67, + 62, + 58, + 67, + 57, + 71, + 73, + 55, + 68, + 74, + 64, + 75, + 68, + 60, + 71, + 64, + 65, + 56, + 70, + 72, + 67, + 67, + 67, + 62, + 75, + 62, + 72, + 72, + 50, + 78, + 83, + 66, + 54, + 63, + 72, + 72, + 66, + 53, + 74, + 57, + 65, + 53, + 61, + 60, + 57, + 43, + 61, + 56, + 59, + 50, + 66, + 56, + 70, + 66, + 60, + 69, + 53, + 64, + 62, + 53, + 57, + 75, + 59, + 55, + 56, + 71, + 68, + 48, + 70, + 59, + 49, + 62, + 60, + 68, + 53, + 71, + 60, + 69, + 66, + 51, + 69, + 63, + 60, + 67, + 65, + 67, + 57, + 59, + 52, + 66, + 63, + 53, + 64, + 53, + 55, + 51, + 65, + 56, + 75, + 70, + 50, + 52, + 54, + 72, + 52, + 61, + 53, + 59, + 69, + 83, + 56, + 66, + 69, + 64, + 49, + 58, + 63, + 56, + 75, + 63, + 62, + 75, + 65, + 77, + 75, + 58, + 55, + 53, + 63, + 58, + 62, + 63, + 68, + 59, + 64, + 61, + 51, + 57, + 61, + 68, + 70, + 62, + 63, + 59, + 49, + 57, + 57, + 65, + 60, + 59, + 63, + 79, + 49, + 51, + 59, + 62, + 75, + 52, + 65, + 61, + 56, + 60, + 78, + 63, + 55, + 72, + 71, + 58, + 56, + 67, + 58, + 49, + 77, + 61, + 74, + 45, + 62, + 69, + 64, + 60, + 55, + 71, + 61, + 69, + 53, + 66, + 63, + 52, + 66, + 57, + 49, + 64, + 61, + 54, + 63, + 53, + 59, + 74, + 70, + 63, + 53, + 52, + 44, + 49, + 52, + 56, + 67, + 42, + 59, + 55, + 66, + 54, + 46, + 51, + 66, + 49, + 52, + 59, + 50, + 55, + 55, + 54, + 71, + 54, + 70, + 56, + 64, + 57, + 51, + 54, + 49, + 60, + 45, + 40, + 55, + 65, + 60, + 69, + 55, + 55, + 57, + 57, + 60, + 51, + 65, + 45, + 46, + 50, + 66, + 44, + 51, + 55, + 57, + 47, + 60, + 50, + 58, + 41, + 49, + 52, + 58, + 49, + 63, + 50, + 52, + 47, + 69, + 72, + 57, + 51, + 57, + 53, + 53, + 62, + 63, + 51, + 50, + 66, + 51, + 68, + 61, + 61, + 45, + 51, + 55, + 50, + 50, + 45, + 57, + 50, + 62, + 56, + 57, + 67, + 46, + 72, + 50, + 57, + 47, + 63, + 57, + 58, + 53, + 45, + 50, + 61, + 48, + 68, + 54, + 62, + 57, + 52, + 43, + 45, + 48, + 48, + 50, + 58, + 53, + 54, + 65, + 44, + 62, + 57, + 45, + 56, + 56, + 44, + 61, + 61, + 62, + 61, + 44, + 64, + 45, + 60, + 44, + 52, + 62, + 53, + 53, + 52, + 49, + 50, + 61, + 57, + 63, + 49, + 61, + 78, + 59, + 57, + 51, + 53, + 58, + 60, + 54, + 58, + 67, + 52, + 45, + 46, + 46, + 48, + 53, + 51, + 57, + 37, + 47, + 44, + 38, + 55, + 57, + 52, + 50, + 39, + 54, + 68, + 49, + 47, + 55, + 48, + 55, + 57, + 57, + 58, + 55, + 64, + 50, + 39, + 41, + 56, + 59, + 52, + 65, + 43, + 44, + 43, + 50, + 62, + 54, + 58, + 56, + 60, + 64, + 42, + 47, + 46, + 35, + 48, + 56, + 39, + 53, + 51, + 47, + 65, + 52, + 64, + 50, + 61, + 51, + 57, + 53, + 48, + 63, + 46, + 42, + 58, + 47, + 37, + 44, + 54, + 49, + 54, + 53, + 41, + 49, + 50, + 45, + 47, + 57, + 68, + 50, + 64, + 50, + 48, + 53, + 55, + 62, + 69, + 51, + 46, + 57, + 57, + 49, + 46, + 40, + 57, + 50, + 47, + 47, + 41, + 52, + 48, + 55, + 42, + 55, + 54, + 50, + 46, + 39, + 52, + 45, + 44, + 60, + 38, + 47, + 40, + 50, + 56, + 48, + 50, + 49, + 46, + 61, + 46, + 50, + 47, + 42, + 44, + 35, + 43, + 46, + 56, + 46, + 49, + 50, + 36, + 44, + 45, + 47, + 51, + 40, + 56, + 49, + 49, + 46, + 56, + 47, + 38, + 49, + 54, + 32, + 68, + 49, + 57, + 40, + 43, + 47, + 43, + 57, + 66, + 51, + 58, + 40, + 38, + 47, + 47, + 48, + 50, + 42, + 40, + 49, + 41, + 40, + 58, + 37, + 51, + 37, + 45, + 53, + 37, + 43, + 50, + 46, + 62, + 47, + 54, + 48, + 33, + 51, + 38, + 40, + 35, + 54, + 41, + 52, + 49, + 43, + 47, + 47, + 58, + 48, + 51, + 44, + 36, + 39, + 51, + 45, + 46, + 50, + 45, + 49, + 46, + 46, + 38, + 64, + 56, + 46, + 44, + 50, + 43, + 42, + 53, + 32, + 44, + 40, + 51, + 52, + 47, + 48, + 59, + 45, + 51, + 51, + 51, + 39, + 59, + 45, + 50, + 33, + 61, + 35, + 32, + 39, + 51, + 53, + 45, + 44, + 46, + 34, + 55, + 41, + 46, + 36, + 53, + 46, + 48, + 43, + 36, + 42, + 37, + 40, + 44, + 35, + 49, + 47, + 42, + 54, + 35, + 41, + 42, + 38, + 39, + 34, + 56, + 48, + 47, + 38, + 31, + 39, + 32, + 37, + 48, + 43, + 46, + 39, + 52, + 48, + 48, + 41, + 51, + 29, + 46, + 39, + 50, + 29, + 33, + 49, + 46, + 46, + 42, + 46, + 33, + 38, + 45, + 48, + 43, + 40, + 50, + 52, + 43, + 45, + 47, + 40, + 52, + 33, + 46, + 47, + 47, + 44, + 46, + 33, + 61, + 49, + 38, + 46, + 41, + 49, + 42, + 44, + 41, + 44, + 52, + 44, + 36, + 45, + 34, + 46, + 33, + 40, + 52, + 58, + 56, + 43, + 63, + 39, + 48, + 55, + 43, + 44, + 44, + 54, + 37, + 51, + 51, + 45, + 36, + 48, + 45, + 48, + 46, + 52, + 46, + 35, + 32, + 45, + 49, + 37, + 30, + 45, + 42, + 45, + 47, + 39, + 35, + 45, + 41, + 48, + 42, + 41, + 46, + 44, + 54, + 43, + 47, + 35, + 40, + 38, + 53, + 48, + 34, + 44, + 37, + 40, + 43, + 46, + 36, + 42, + 54, + 42, + 48, + 35, + 46, + 50, + 37, + 51, + 45, + 44, + 44, + 35, + 33, + 46, + 43, + 38, + 42, + 44, + 39, + 37, + 43, + 42, + 45, + 40, + 45, + 51, + 37, + 41, + 54, + 43, + 46, + 45, + 38, + 39, + 32, + 43, + 39, + 39, + 42, + 44, + 36, + 40, + 41, + 35, + 36, + 49, + 28, + 50, + 48, + 50, + 45, + 37, + 35, + 36, + 44, + 39, + 45, + 46, + 37, + 47, + 39, + 45, + 38, + 50, + 38, + 47, + 36, + 35, + 31, + 31, + 51, + 42, + 34, + 45, + 51, + 27, + 41, + 29, + 31, + 43, + 45, + 45, + 41, + 40, + 34, + 39, + 35, + 33, + 41, + 33, + 35, + 42, + 40, + 33, + 41, + 37, + 35, + 41, + 34, + 40, + 45, + 36, + 36, + 39, + 44, + 42, + 38, + 45, + 37, + 42, + 38, + 46, + 43, + 35, + 44, + 38, + 35, + 45, + 56, + 32, + 47, + 39, + 37, + 33, + 36, + 49, + 31, + 45, + 31, + 30, + 42, + 44, + 33, + 39, + 28, + 39, + 34, + 37, + 39, + 39, + 43, + 38, + 44, + 41, + 46, + 41, + 42, + 42, + 45, + 29, + 44, + 32, + 39, + 50, + 41, + 33, + 45, + 42, + 35, + 33, + 35, + 32, + 44, + 37, + 32, + 35, + 38, + 25, + 38, + 43, + 33, + 43, + 40, + 37, + 30, + 40, + 37, + 37, + 41, + 33, + 45, + 53, + 35, + 33, + 34, + 44, + 36, + 51, + 41, + 32, + 34, + 43, + 34, + 38, + 33, + 50, + 50, + 44, + 44, + 49, + 41, + 37, + 34, + 42, + 39, + 43, + 39, + 35, + 36, + 43, + 35, + 38, + 38, + 59, + 36, + 35, + 50, + 28, + 38, + 38, + 33, + 34, + 39, + 48, + 35, + 47, + 33, + 39, + 37, + 34, + 37, + 29, + 30, + 47, + 34, + 29, + 35, + 25, + 39, + 34, + 40, + 44, + 41, + 42, + 59, + 48, + 29, + 26, + 33, + 39, + 29, + 40, + 40, + 34, + 41, + 39, + 29, + 31, + 38, + 37, + 31, + 37, + 31, + 30, + 46, + 23, + 32, + 25, + 38, + 32, + 37, + 52, + 38, + 42, + 38, + 22, + 37, + 33, + 33, + 36, + 33, + 22, + 29, + 43, + 31, + 38, + 32, + 36, + 33, + 37, + 32, + 30, + 29, + 52, + 44, + 26, + 33, + 37, + 46, + 26, + 27, + 40, + 34, + 30, + 34, + 33, + 37, + 27, + 34, + 28, + 31, + 35, + 29, + 28, + 38, + 39, + 37, + 34, + 33, + 41, + 40, + 24, + 25, + 34, + 35, + 35, + 36, + 33, + 42, + 29, + 35, + 35, + 39, + 28, + 32, + 25, + 26, + 38, + 31, + 39, + 26, + 30, + 42, + 38, + 44, + 33, + 33, + 38, + 35, + 41, + 27, + 33, + 37, + 38, + 22, + 43, + 27, + 35, + 32, + 41, + 23, + 31, + 28, + 34, + 43, + 32, + 35, + 32, + 47, + 36, + 35, + 33, + 20, + 42, + 36, + 35, + 28, + 37, + 34, + 36, + 30, + 38, + 33, + 32, + 37, + 47, + 35, + 34, + 29, + 28, + 38, + 41, + 37, + 27, + 46, + 41, + 29, + 37, + 40, + 35, + 35, + 24, + 31, + 45, + 30, + 37, + 34, + 32, + 28, + 31, + 30, + 44, + 40, + 37, + 38, + 30, + 38, + 35, + 27, + 37, + 33, + 34, + 18, + 34, + 25, + 36, + 38, + 26, + 21, + 21, + 36, + 34, + 33, + 24, + 23, + 39, + 39, + 49, + 41, + 26, + 28, + 29, + 33, + 32, + 26, + 13, + 35, + 37, + 44, + 44, + 35, + 38, + 23, + 31, + 27, + 26, + 25, + 32, + 31, + 37, + 27, + 21, + 35, + 29, + 31, + 27, + 34, + 19, + 26, + 37, + 30, + 34, + 39, + 30, + 23, + 34, + 33, + 34, + 27, + 36, + 26, + 27, + 31, + 31, + 29, + 31, + 43, + 34, + 32, + 28, + 34, + 40, + 40, + 42, + 32, + 30, + 33, + 35, + 33, + 40, + 34, + 39, + 30, + 32, + 36, + 27, + 28, + 30, + 30, + 35, + 28, + 26, + 40, + 35, + 25, + 31, + 38, + 24, + 32, + 32, + 39, + 30, + 26, + 38, + 39, + 25, + 29, + 34, + 34, + 23, + 32, + 33, + 35, + 23, + 29, + 26, + 29, + 33, + 31, + 23, + 26, + 31, + 37, + 26, + 37, + 30, + 24, + 29, + 33, + 27, + 34, + 43, + 32, + 33, + 35, + 30, + 31, + 27, + 32, + 37, + 23, + 40, + 33, + 23, + 32, + 29, + 25, + 42, + 28, + 37, + 35, + 34, + 24, + 30, + 27, + 32, + 32, + 24, + 23, + 33, + 30, + 30, + 25, + 23, + 26, + 27, + 30, + 35, + 35, + 30, + 30, + 28, + 31, + 21, + 32, + 29, + 35, + 31, + 36, + 27, + 20, + 28, + 29, + 24, + 21, + 21, + 38, + 22, + 28, + 30, + 36, + 29, + 31, + 31, + 29, + 35, + 18, + 24, + 29, + 41, + 30, + 25, + 27, + 35, + 25, + 28, + 29, + 33, + 34, + 33, + 31, + 37, + 23, + 25, + 28, + 23, + 24, + 39, + 40, + 25, + 33, + 36, + 27, + 27, + 37, + 31, + 27, + 31, + 30, + 24, + 26, + 25, + 28, + 24, + 34, + 25, + 33, + 23, + 38, + 25, + 32, + 30, + 32, + 31, + 21, + 31, + 22, + 28, + 40, + 29, + 31, + 29, + 32, + 35, + 26, + 24, + 23, + 28, + 29, + 28, + 37, + 39, + 31, + 30, + 29, + 38, + 38, + 38, + 28, + 28, + 35, + 28, + 32, + 32, + 24, + 20, + 27, + 29, + 20, + 29, + 25, + 27, + 40, + 26, + 24, + 26, + 27, + 28, + 30, + 32, + 20, + 34, + 32, + 31, + 38, + 30, + 27, + 29, + 30, + 22, + 32, + 23, + 29, + 26, + 21, + 23, + 31, + 29, + 32, + 21, + 24, + 31, + 29, + 30, + 34, + 26, + 28, + 35, + 14, + 28, + 26, + 24, + 22, + 26, + 23, + 30, + 27, + 24, + 27, + 30, + 22, + 25, + 19, + 25, + 24, + 29, + 35, + 31, + 27, + 36, + 24, + 20, + 35, + 28, + 33, + 30, + 24, + 26, + 28, + 23, + 16, + 21, + 32, + 41, + 31, + 29, + 38, + 21, + 15, + 24, + 31, + 25, + 29, + 27, + 36, + 33, + 28, + 31, + 24, + 31, + 24, + 22, + 20, + 31, + 34, + 30, + 23, + 22, + 24, + 29, + 38, + 21, + 23, + 23, + 38, + 24, + 30, + 31, + 27, + 27, + 24, + 20, + 29, + 21, + 30, + 26, + 18, + 35, + 25, + 28, + 31, + 23, + 24, + 28, + 32, + 21, + 33, + 17, + 20, + 20, + 24, + 24, + 27, + 28, + 27, + 30, + 24, + 29, + 31, + 25, + 32, + 22, + 19, + 34, + 32, + 31, + 15, + 25, + 28, + 24, + 37, + 23, + 25, + 36, + 32, + 28, + 24, + 36, + 32, + 17, + 19, + 21, + 26, + 21, + 31, + 32, + 32, + 29, + 29, + 25, + 25, + 26, + 27, + 25, + 27, + 26, + 20, + 22, + 34, + 29, + 22, + 30, + 28, + 31, + 23, + 23, + 22, + 26, + 26, + 33, + 33, + 26, + 27, + 25, + 38, + 19, + 32, + 21, + 25, + 31, + 21, + 15, + 20, + 17, + 34, + 26, + 20, + 35, + 30, + 31, + 27, + 27, + 27, + 28, + 22, + 18, + 26, + 27, + 21, + 26, + 38, + 27, + 36, + 26, + 26, + 31, + 28, + 18, + 25, + 22, + 23, + 25, + 29, + 25, + 31, + 23, + 23, + 25, + 28, + 20, + 25, + 32, + 27, + 25, + 17, + 23, + 19, + 27, + 27, + 21, + 27, + 30, + 23, + 33, + 32, + 25, + 22, + 22, + 29, + 23, + 22, + 26, + 25, + 19, + 28, + 20, + 25, + 35, + 17, + 34, + 30, + 29, + 18, + 26, + 23, + 28, + 31, + 31, + 26, + 19, + 27, + 20, + 30, + 26, + 14, + 24, + 27, + 22, + 26, + 22, + 25, + 22, + 32, + 23, + 31, + 26, + 28, + 21, + 21, + 34, + 31, + 29, + 25, + 21, + 23, + 26, + 25, + 23, + 25, + 23, + 23, + 25, + 22, + 22, + 27, + 26, + 26, + 27, + 26, + 28, + 22, + 30, + 25, + 27, + 38, + 24, + 27, + 27, + 23, + 25, + 20, + 27, + 31, + 25, + 15, + 34, + 25, + 27, + 34, + 21, + 19, + 17, + 27, + 26, + 20, + 27, + 16, + 16, + 27, + 24, + 25, + 26, + 25, + 24, + 19, + 21, + 21, + 30, + 20, + 23, + 28, + 31, + 24, + 24, + 28, + 31, + 22, + 23, + 21, + 25, + 24, + 19, + 25, + 30, + 18, + 23, + 29, + 16, + 27, + 21, + 16, + 30, + 28, + 25, + 24, + 20, + 25, + 23, + 24, + 21, + 31, + 27, + 15, + 18, + 21, + 18, + 21, + 32, + 33, + 21, + 24, + 22, + 29, + 26, + 28, + 22, + 25, + 22, + 23, + 28, + 28, + 21, + 28, + 21, + 21, + 20, + 19, + 36, + 19, + 22, + 34, + 26, + 29, + 17, + 28, + 22, + 20, + 20, + 24, + 25, + 21, + 21, + 27, + 27, + 27, + 26, + 22, + 22, + 15, + 16, + 25, + 17, + 16, + 23, + 36, + 23, + 32, + 22, + 23, + 22, + 21, + 18, + 31, + 23, + 15, + 28, + 14, + 24, + 25, + 14, + 29, + 19, + 20, + 26, + 27, + 19, + 20, + 24, + 24, + 21, + 29, + 23, + 21, + 22, + 34, + 28, + 29, + 37, + 22, + 28, + 16, + 25, + 22, + 25, + 25, + 15, + 24, + 31, + 24, + 12, + 11, + 20, + 26, + 19, + 20, + 25, + 15, + 18, + 26, + 17, + 19, + 23, + 24, + 21, + 16, + 19, + 25, + 28, + 17, + 21, + 18, + 19, + 24, + 28, + 16, + 30, + 27, + 18, + 21, + 27, + 23, + 14, + 26, + 15, + 22, + 35, + 26, + 26, + 24, + 16, + 21, + 18, + 16, + 10, + 23, + 27, + 19, + 25, + 24, + 22, + 20, + 22, + 26, + 29, + 19, + 18, + 22, + 25, + 22, + 30, + 16, + 18, + 19, + 22, + 18, + 17, + 20, + 18, + 17, + 27, + 20, + 22, + 19, + 17, + 22, + 16, + 27, + 25, + 30, + 24, + 20, + 25, + 19, + 22, + 24, + 21, + 16, + 20, + 18, + 32, + 21, + 25, + 32, + 20, + 21, + 11, + 16, + 27, + 21, + 18, + 22, + 20, + 27, + 21, + 27, + 13, + 19, + 25, + 13, + 17, + 17, + 20, + 26, + 25, + 9, + 28, + 18, + 19, + 23, + 31, + 23, + 19, + 25, + 25, + 20, + 24, + 20, + 23, + 19, + 28, + 19, + 23, + 16, + 16, + 21, + 19, + 19, + 22, + 22, + 21, + 19, + 23, + 23, + 29, + 25, + 20, + 16, + 22, + 20, + 32, + 21, + 28, + 13, + 15, + 27, + 20, + 33, + 14, + 24, + 18, + 28, + 16, + 18, + 27, + 25, + 23, + 16, + 26, + 20, + 26, + 26, + 17, + 20, + 18, + 29, + 27, + 22, + 33, + 28, + 23, + 20, + 24, + 17, + 14, + 17, + 26, + 22, + 15, + 22, + 27, + 9, + 12, + 16, + 22, + 13, + 20, + 16, + 16, + 23, + 22, + 20, + 27, + 23, + 19, + 26, + 18, + 16, + 19, + 18, + 20, + 14, + 24, + 21, + 19, + 13, + 10, + 20, + 28, + 26, + 26, + 17, + 14, + 15, + 26, + 20, + 18, + 18, + 22, + 25, + 25, + 25, + 23, + 19, + 20, + 20, + 19, + 19, + 19, + 19, + 22, + 23, + 25, + 22, + 31, + 18, + 20, + 16, + 20, + 23, + 29, + 21, + 24, + 22, + 25, + 18, + 15, + 21, + 28, + 26, + 20, + 23, + 17, + 18, + 19, + 24, + 15, + 23, + 19, + 13, + 22, + 18, + 22, + 18, + 20, + 18, + 19, + 19, + 27, + 17, + 18, + 22, + 18, + 20, + 20, + 23, + 14, + 19, + 17, + 15, + 31, + 21, + 31, + 19, + 17, + 18, + 19, + 20, + 13, + 15, + 21, + 24, + 20, + 16, + 18, + 17, + 18, + 20, + 20, + 24, + 20, + 21, + 16, + 22, + 15, + 18, + 21, + 15, + 20, + 21, + 20, + 22, + 21, + 21, + 19, + 17, + 19, + 13, + 12, + 22, + 16, + 20, + 19, + 23, + 18, + 15, + 16, + 21, + 22, + 10, + 14, + 19, + 26, + 16, + 18, + 16, + 17, + 19, + 15, + 22, + 24, + 10, + 15, + 15, + 19, + 20, + 20, + 17, + 19, + 18, + 21, + 12, + 18, + 21, + 15, + 20, + 24, + 15, + 25, + 15, + 25, + 14, + 13, + 17, + 23, + 18, + 18, + 31, + 20, + 23, + 13, + 23, + 21, + 11, + 21, + 15, + 18, + 19, + 19, + 14, + 21, + 22, + 13, + 16, + 14, + 18, + 16, + 29, + 25, + 24, + 20, + 17, + 18, + 16, + 19, + 16, + 19, + 19, + 14, + 17, + 13, + 25, + 17, + 18, + 26, + 20, + 22, + 19, + 20, + 12, + 21, + 19, + 18, + 24, + 16, + 26, + 31, + 20, + 16, + 20, + 23, + 24, + 21, + 18, + 17, + 16, + 11, + 13, + 17, + 15, + 18, + 20, + 16, + 22, + 24, + 15, + 22, + 15, + 17, + 16, + 22, + 13, + 19, + 23, + 25, + 23, + 19, + 23, + 16, + 20, + 22, + 19, + 17, + 22, + 18, + 16, + 15, + 24, + 21, + 14, + 12, + 24, + 15, + 15, + 24, + 19, + 21, + 19, + 13, + 19, + 24, + 25, + 25, + 21, + 23, + 15, + 18, + 15, + 12, + 17, + 10, + 18, + 22, + 24, + 26, + 21, + 21, + 10, + 18, + 25, + 24, + 11, + 17, + 16, + 18, + 24, + 21, + 19, + 15, + 15, + 21, + 21, + 22, + 22, + 16, + 22, + 21, + 15, + 16, + 18, + 19, + 14, + 24, + 20, + 18, + 17, + 21, + 20, + 10, + 14, + 18, + 19, + 12, + 12, + 23, + 18, + 20, + 14, + 25, + 29, + 23, + 15, + 21, + 19, + 13, + 23, + 12, + 19, + 14, + 22, + 9, + 18, + 15, + 20, + 16, + 27, + 15, + 19, + 20, + 16, + 20, + 15, + 16, + 17, + 15, + 23, + 26, + 14, + 23, + 18, + 21, + 28, + 21, + 20, + 21, + 19, + 16, + 18, + 25, + 16, + 13, + 16, + 22, + 17, + 18, + 14, + 18, + 18, + 27, + 18, + 17, + 17, + 17, + 19, + 29, + 22, + 16, + 18, + 21, + 23, + 11, + 16, + 21, + 15, + 21, + 16, + 20, + 23, + 16, + 14, + 11, + 19, + 18, + 18, + 12, + 25, + 16, + 11, + 16, + 10, + 12, + 19, + 14, + 21, + 17, + 13, + 20, + 21, + 13, + 21, + 15, + 16, + 14, + 20, + 10, + 15, + 16, + 11, + 13, + 12, + 14, + 17, + 20, + 19, + 15, + 27, + 16, + 19, + 21, + 20, + 21, + 24, + 12, + 18, + 22, + 24, + 13, + 22, + 22, + 16, + 18, + 17, + 12, + 21, + 22, + 21, + 18, + 17, + 20, + 19, + 18, + 21, + 13, + 11, + 11, + 18, + 24, + 15, + 17, + 17, + 18, + 15, + 13, + 17, + 16, + 15, + 19, + 16, + 19, + 16, + 14, + 16, + 15, + 16, + 19, + 16, + 23, + 17, + 12, + 12, + 18, + 18, + 22, + 12, + 15, + 12, + 22, + 19, + 15, + 19, + 16, + 14, + 8, + 18, + 12, + 15, + 17, + 18, + 22, + 12, + 22, + 14, + 19, + 21, + 16, + 18, + 17, + 19, + 17, + 23, + 22, + 21, + 23, + 19, + 15, + 13, + 30, + 19, + 14, + 10, + 15, + 18, + 24, + 16, + 14, + 15, + 17, + 25, + 16, + 15, + 12, + 23, + 19, + 20, + 13, + 19, + 21, + 16, + 22, + 21, + 25, + 14, + 21, + 13, + 12, + 20, + 12, + 21, + 16, + 18, + 19, + 22, + 14, + 15, + 20, + 13, + 20, + 12, + 12, + 16, + 10, + 16, + 15, + 16, + 14, + 16, + 17, + 16, + 14, + 16, + 17, + 10, + 22, + 17, + 18, + 15, + 15, + 18, + 19, + 19, + 10, + 20, + 18, + 18, + 17, + 19, + 16, + 13, + 18, + 15, + 16, + 15, + 17, + 8, + 20, + 19, + 11, + 23, + 13, + 19, + 16, + 15, + 17, + 20, + 16, + 17, + 11, + 10, + 17, + 20, + 22, + 13, + 19, + 14, + 23, + 17, + 14, + 24, + 27, + 20, + 17, + 14, + 19, + 16, + 17, + 13, + 14, + 14, + 17, + 18, + 13, + 27, + 13, + 15, + 11, + 12, + 22, + 17, + 17, + 10, + 12, + 13, + 11, + 20, + 14, + 20, + 15, + 10, + 8, + 15, + 15, + 12, + 13, + 17, + 15, + 12, + 20, + 16, + 17, + 14, + 29, + 16, + 18, + 13, + 23, + 8, + 12, + 20, + 11, + 14, + 18, + 16, + 11, + 22, + 13, + 16, + 15, + 16, + 14, + 17, + 24, + 18, + 21, + 13, + 15, + 19, + 14, + 12, + 16, + 14, + 17, + 21, + 12, + 16, + 21, + 10, + 18, + 17, + 18, + 14, + 14, + 16, + 15, + 14, + 16, + 17, + 20, + 18, + 16, + 13, + 13, + 16, + 18, + 16, + 15, + 20, + 14, + 19, + 19, + 24, + 14, + 12, + 13, + 12, + 18, + 15, + 17, + 14, + 16, + 13, + 20, + 21, + 13, + 9, + 14, + 18, + 17, + 11, + 19, + 11, + 13, + 17, + 16, + 20, + 21, + 13, + 10, + 12, + 14, + 14, + 17, + 20, + 16, + 13, + 13, + 17, + 13, + 16, + 17, + 20, + 15, + 12, + 20, + 17, + 17, + 14, + 14, + 15, + 13, + 19, + 14, + 15, + 16, + 18, + 15, + 16, + 13, + 18, + 15, + 13, + 11, + 21, + 16, + 11, + 21, + 16, + 19, + 15, + 15, + 17, + 14, + 16, + 13, + 17, + 15, + 18, + 16, + 25, + 15, + 13, + 19, + 12, + 15, + 12, + 10, + 12, + 11, + 19, + 15, + 19, + 15, + 11, + 15, + 15, + 18, + 9, + 14, + 19, + 6, + 17, + 22, + 10, + 15, + 15, + 15, + 19, + 17, + 9, + 15, + 12, + 22, + 16, + 12, + 16, + 9, + 18, + 17, + 24, + 16, + 19, + 13, + 16, + 20, + 22, + 14, + 12, + 11, + 8, + 12, + 19, + 14, + 15, + 18, + 13, + 13, + 10, + 22, + 17, + 15, + 16, + 20, + 16, + 16, + 13, + 24, + 7, + 12, + 10, + 16, + 17, + 10, + 16, + 12, + 12, + 10, + 16, + 17, + 12, + 9, + 13, + 16, + 10, + 12, + 11, + 14, + 15, + 11, + 13, + 9, + 18, + 18, + 12, + 7, + 11, + 15, + 16, + 15, + 8, + 16, + 14, + 14, + 17, + 21, + 20, + 12, + 29, + 22, + 12, + 12, + 14, + 18, + 9, + 13, + 11, + 14, + 12, + 16, + 11, + 9, + 15, + 16, + 20, + 14, + 17, + 16, + 9, + 13, + 13, + 10, + 14, + 11, + 13, + 13, + 12, + 12, + 13, + 14, + 14, + 7, + 20, + 21, + 12, + 16, + 19, + 11, + 20, + 13, + 10, + 11, + 10, + 14, + 10, + 17, + 15, + 17, + 19, + 15, + 17, + 16, + 16, + 21, + 13, + 13, + 12, + 10, + 16, + 20, + 11, + 11, + 13, + 12, + 22, + 12, + 9, + 13, + 13, + 13, + 16, + 12, + 18, + 12, + 7, + 20, + 10, + 16, + 14, + 13, + 11, + 17, + 10, + 18, + 9, + 14, + 17, + 13, + 16, + 13, + 16, + 19, + 14, + 20, + 13, + 13, + 14, + 11, + 25, + 14, + 21, + 17, + 10, + 14, + 19, + 16, + 18, + 13, + 19, + 15, + 11, + 12, + 9, + 18, + 11, + 16, + 13, + 10, + 12, + 11, + 13, + 13, + 9, + 13, + 22, + 12, + 11, + 19, + 15, + 12, + 19, + 17, + 18, + 13, + 11, + 15, + 16, + 8, + 15, + 15, + 13, + 19, + 15, + 13, + 12, + 8, + 12, + 12, + 12, + 12, + 9, + 21, + 9, + 13, + 17, + 9, + 8, + 23, + 15, + 11, + 16, + 18, + 11, + 12, + 13, + 14, + 14, + 12, + 16, + 11, + 17, + 20, + 15, + 22, + 14, + 13, + 13, + 10, + 16, + 12, + 13, + 15, + 14, + 18, + 11, + 24, + 13, + 10, + 10, + 16, + 13, + 11, + 14, + 16, + 10, + 11, + 19, + 16, + 11, + 10, + 11, + 12, + 9, + 12, + 10, + 10, + 8, + 20, + 12, + 9, + 9, + 20, + 13, + 15, + 9, + 21, + 18, + 14, + 8, + 10, + 17, + 8, + 12, + 20, + 15, + 13, + 14, + 13, + 14, + 16, + 7, + 11, + 11, + 14, + 11, + 11, + 11, + 14, + 13, + 12, + 8, + 16, + 8, + 10, + 13, + 13, + 18, + 15, + 5, + 10, + 15, + 18, + 9, + 13, + 14, + 15, + 13, + 14, + 19, + 9, + 8, + 10, + 12, + 12, + 11, + 11, + 16, + 11, + 9, + 16, + 9, + 10, + 7, + 15, + 22, + 10, + 11, + 4, + 10, + 12, + 11, + 13, + 15, + 7, + 13, + 12, + 12, + 17, + 11, + 12, + 17, + 12, + 10, + 13, + 12, + 11, + 11, + 12, + 15, + 15, + 11, + 9, + 7, + 17, + 15, + 12, + 21, + 18, + 15, + 14, + 19, + 8, + 7, + 17, + 8, + 20, + 19, + 8, + 14, + 11, + 8, + 10, + 9, + 15, + 9, + 21, + 10, + 18, + 13, + 12, + 18, + 13, + 11, + 8, + 15, + 12, + 10, + 12, + 14, + 11, + 9, + 20, + 8, + 12, + 11, + 10, + 18, + 16, + 20, + 12, + 11, + 10, + 11, + 10, + 3, + 13, + 11, + 16, + 8, + 5, + 14, + 13, + 17, + 19, + 10, + 9, + 17, + 11, + 11, + 9, + 17, + 10, + 6, + 12, + 7, + 9, + 13, + 17, + 12, + 11, + 18, + 13, + 6, + 14, + 8, + 15, + 13, + 10, + 13, + 13, + 14, + 10, + 12, + 17, + 10, + 16, + 26, + 5, + 14, + 13, + 15, + 11, + 11, + 14, + 10, + 14, + 17, + 15, + 17, + 12, + 14, + 15, + 9, + 15, + 16, + 10, + 10, + 14, + 16, + 11, + 8, + 14, + 11, + 15, + 8, + 21, + 9, + 16, + 13, + 13, + 14, + 8, + 12, + 16, + 15, + 12, + 10, + 11, + 14, + 11, + 18, + 14, + 19, + 9, + 17, + 14, + 15, + 13, + 10, + 12, + 12, + 11, + 7, + 12, + 16, + 13, + 16, + 7, + 11, + 12, + 13, + 14, + 7, + 12, + 10, + 14, + 10, + 5, + 9, + 8, + 14, + 14, + 5, + 9, + 14, + 15, + 6, + 13, + 17, + 11, + 9, + 14, + 13, + 14, + 21, + 17, + 9, + 12, + 14, + 8, + 10, + 12, + 6, + 10, + 11, + 11, + 13, + 9, + 12, + 13, + 12, + 14, + 10, + 17, + 10, + 7, + 8, + 16, + 14, + 12, + 22, + 12, + 14, + 17, + 14, + 11, + 20, + 12, + 13, + 16, + 14, + 14, + 10, + 9, + 14, + 12, + 12, + 13, + 10, + 9, + 12, + 6, + 8, + 11, + 7, + 10, + 7, + 5, + 10, + 10, + 13, + 16, + 5, + 11, + 12, + 9, + 13, + 10, + 11, + 13, + 19, + 9, + 9, + 7, + 10, + 8, + 10, + 8, + 15, + 4, + 7, + 8, + 11, + 10, + 17, + 13, + 11, + 15, + 13, + 10, + 7, + 5, + 13, + 13, + 7, + 9, + 7, + 13, + 12, + 7, + 14, + 8, + 12, + 11, + 6, + 14, + 16, + 8, + 12, + 11, + 12, + 7, + 10, + 20, + 8, + 7, + 11, + 14, + 9, + 9, + 16, + 17, + 9, + 7, + 13, + 14, + 12, + 6, + 9, + 8, + 14, + 7, + 10, + 6, + 15, + 9, + 14, + 19, + 10, + 11, + 12, + 18, + 17, + 9, + 12, + 11, + 9, + 4, + 4, + 11, + 14, + 8, + 12, + 7, + 14, + 9, + 10, + 14, + 5, + 13, + 14, + 14, + 9, + 8, + 10, + 10, + 10, + 11, + 8, + 17, + 10, + 8, + 10, + 10, + 8, + 13, + 9, + 10, + 6, + 12, + 17, + 11, + 6, + 13, + 5, + 12, + 14, + 10, + 9, + 11, + 15, + 9, + 11, + 21, + 8, + 5, + 5, + 8, + 12, + 18, + 13, + 12, + 8, + 7, + 6, + 14, + 14, + 15, + 14, + 10, + 14, + 11, + 16, + 9, + 10, + 11, + 12, + 8, + 11, + 11, + 11, + 16, + 6, + 14, + 9, + 7, + 11, + 11, + 10, + 13, + 12, + 7, + 11, + 4, + 5, + 8, + 12, + 5, + 12, + 9, + 10, + 10, + 13, + 9, + 9, + 7, + 9, + 9, + 10, + 6, + 16, + 7, + 4, + 7, + 15, + 12, + 8, + 15, + 7, + 17, + 14, + 12, + 12, + 10, + 14, + 16, + 14, + 7, + 9, + 11, + 8, + 11, + 8, + 10, + 13, + 8, + 8, + 11, + 10, + 12, + 9, + 6, + 10, + 7, + 12, + 10, + 11, + 11, + 11, + 12, + 13, + 10, + 9, + 7, + 15, + 12, + 10, + 7, + 8, + 13, + 7, + 8, + 5, + 9, + 9, + 8, + 10, + 7, + 11, + 11, + 13, + 11, + 9, + 9, + 10, + 14, + 9, + 14, + 10, + 14, + 12, + 12, + 6, + 12, + 6, + 13, + 9, + 5, + 13, + 10, + 16, + 7, + 11, + 11, + 7, + 14, + 4, + 10, + 9, + 7, + 12, + 17, + 22, + 9, + 12, + 7, + 10, + 10, + 9, + 9, + 15, + 9, + 7, + 7, + 11, + 13, + 10, + 11, + 13, + 6, + 10, + 14, + 8, + 7, + 14, + 7, + 11, + 16, + 5, + 10, + 13, + 8, + 9, + 4, + 10, + 8, + 13, + 7, + 9, + 10, + 8, + 6, + 2, + 10, + 9, + 6, + 11, + 13, + 10, + 11, + 8, + 11, + 3, + 22, + 14, + 8, + 6, + 10, + 8, + 10, + 10, + 5, + 11, + 12, + 8, + 4, + 18, + 13, + 11, + 8, + 7, + 12, + 6, + 8, + 9, + 10, + 6, + 9, + 11, + 8, + 10, + 2, + 12, + 9, + 12, + 9, + 11, + 15, + 7, + 11, + 13, + 16, + 11, + 10, + 13, + 7, + 11, + 12, + 15, + 6, + 11, + 9, + 7, + 12, + 11, + 16, + 13, + 16, + 8, + 10, + 14, + 12, + 11, + 13, + 8, + 7, + 5, + 6, + 9, + 6, + 8, + 5, + 11, + 11, + 10, + 5, + 17, + 13, + 6, + 9, + 10, + 12, + 12, + 12, + 8, + 9, + 4, + 11, + 10, + 10, + 8, + 7, + 9, + 13, + 5, + 7, + 15, + 9, + 12, + 7, + 5, + 6, + 12, + 5, + 9, + 11, + 10, + 7, + 12, + 6, + 11, + 9, + 7, + 6, + 9, + 7, + 12, + 6, + 4, + 10, + 8, + 5, + 16, + 14, + 7, + 7, + 9, + 11, + 9, + 6, + 12, + 10, + 7, + 11, + 5, + 6, + 12, + 11, + 7, + 4, + 10, + 14, + 11, + 13, + 10, + 8, + 9, + 6, + 7, + 11, + 10, + 12, + 11, + 12, + 14, + 5, + 11, + 15, + 11, + 7, + 6, + 6, + 8, + 9, + 9, + 9, + 12, + 10, + 10, + 11, + 11, + 10, + 7, + 12, + 12, + 11, + 13, + 7, + 7, + 10, + 12, + 7, + 9, + 6, + 11, + 11, + 9, + 11, + 4, + 12, + 9, + 8, + 12, + 10, + 10, + 9, + 8, + 3, + 11, + 8, + 10, + 11, + 18, + 8, + 5, + 9, + 13, + 11, + 10, + 8, + 5, + 8, + 11, + 9, + 13, + 10, + 5, + 6, + 9, + 11, + 3, + 8, + 9, + 12, + 11, + 6, + 9, + 9, + 8, + 5, + 7, + 11, + 8, + 8, + 11, + 7, + 9, + 8, + 10, + 8, + 7, + 15, + 7, + 12, + 8, + 12, + 8, + 4, + 6, + 8, + 16, + 8, + 15, + 6, + 11, + 14, + 4, + 13, + 9, + 4, + 13, + 8, + 6, + 14, + 9, + 9, + 6, + 11, + 7, + 17, + 5, + 10, + 6, + 9, + 8, + 13, + 6, + 18, + 9, + 5, + 8, + 6, + 8, + 9, + 12, + 11, + 10, + 12, + 7, + 6, + 8, + 10, + 11, + 12, + 7, + 5, + 13, + 13, + 3, + 8, + 7, + 6, + 7, + 4, + 7, + 9, + 13, + 10, + 5, + 7, + 11, + 11, + 11, + 3, + 6, + 10, + 2, + 9, + 9, + 5, + 2, + 5, + 12, + 9, + 3, + 11, + 11, + 11, + 11, + 3, + 11, + 10, + 6, + 12, + 17, + 8, + 6, + 13, + 10, + 5, + 5, + 8, + 8, + 8, + 13, + 9, + 10, + 5, + 6, + 15, + 4, + 8, + 10, + 10, + 9, + 11, + 6, + 6, + 7, + 11, + 6, + 9, + 15, + 5, + 6, + 12, + 11, + 5, + 14, + 8, + 15, + 7, + 12, + 7, + 10, + 6, + 9, + 8, + 9, + 7, + 10, + 13, + 6, + 11, + 10, + 7, + 6, + 11, + 15, + 7, + 4, + 7, + 7, + 9, + 9, + 9, + 5, + 6, + 10, + 14, + 7, + 13, + 8, + 11, + 2, + 5, + 6, + 8, + 12, + 8, + 9, + 5, + 8, + 6, + 9, + 11, + 5, + 6, + 8, + 5, + 8, + 9, + 13, + 8, + 7, + 9, + 8, + 11, + 6, + 9, + 10, + 16, + 7, + 5, + 6, + 9, + 15, + 9, + 7, + 10, + 7, + 7, + 8, + 11, + 6, + 10, + 9, + 12, + 12, + 8, + 7, + 8, + 7, + 12, + 8, + 16, + 8, + 10, + 9, + 5, + 6, + 6, + 10, + 12, + 9, + 8, + 6, + 6, + 6, + 5, + 9, + 10, + 16, + 9, + 13, + 8, + 6, + 9, + 10, + 15, + 2, + 3, + 10, + 14, + 8, + 7, + 11, + 5, + 5, + 6, + 8, + 8, + 8, + 15, + 8, + 12, + 13, + 7, + 11, + 9, + 8, + 11, + 5, + 5, + 6, + 10, + 12, + 5, + 7, + 8, + 9, + 7, + 5, + 7, + 14, + 8, + 16, + 4, + 14, + 11, + 5, + 4, + 7, + 16, + 6, + 10, + 6, + 8, + 12, + 5, + 7, + 9, + 8, + 5, + 10, + 7, + 8, + 15, + 9, + 7, + 8, + 12, + 5, + 10, + 7, + 7, + 8, + 10, + 4, + 8, + 6, + 7, + 13, + 5, + 6, + 7, + 4, + 6, + 12, + 12, + 8, + 9, + 8, + 6, + 8, + 10, + 7, + 13, + 3, + 9, + 10, + 7, + 5, + 6, + 4, + 3, + 7, + 14, + 6, + 9, + 11, + 10, + 6, + 6, + 3, + 8, + 7, + 6, + 4, + 7, + 8, + 9, + 5, + 12, + 12, + 7, + 7, + 11, + 7, + 8, + 5, + 7, + 11, + 7, + 6, + 4, + 7, + 11, + 4, + 7, + 9, + 6, + 13, + 6, + 10, + 7, + 8, + 6, + 6, + 8, + 7, + 9, + 7, + 8, + 5, + 12, + 9, + 4, + 13, + 11, + 6, + 8, + 5, + 6, + 7, + 8, + 7, + 12, + 7, + 9, + 8, + 7, + 5, + 5, + 11, + 7, + 9, + 7, + 5, + 9, + 8, + 6, + 10, + 6, + 8, + 6, + 7, + 11, + 13, + 3, + 10, + 6, + 3, + 8, + 3, + 10, + 9, + 8, + 11, + 5, + 12, + 10, + 9, + 10, + 2, + 10, + 7, + 9, + 10, + 9, + 8, + 8, + 6, + 7, + 8, + 12, + 9, + 10, + 4, + 12, + 7, + 5, + 9, + 12, + 6, + 7, + 6, + 10, + 2, + 10, + 13, + 9, + 7, + 5, + 11, + 5, + 13, + 10, + 2, + 4, + 4, + 4, + 8, + 8, + 3, + 9, + 8, + 3, + 12, + 10, + 5, + 8, + 10, + 9, + 14, + 6, + 3, + 7, + 10, + 8, + 7, + 6, + 7, + 9, + 5, + 6, + 7, + 4, + 9, + 10, + 12, + 7, + 2, + 7, + 8, + 5, + 11, + 12, + 12, + 6, + 8, + 5, + 7, + 6, + 10, + 5, + 8, + 9, + 14, + 5, + 4, + 6, + 8, + 9, + 3, + 11, + 9, + 10, + 12, + 1, + 10, + 12, + 8, + 4, + 10, + 4, + 8, + 12, + 12, + 7, + 8, + 7, + 13, + 4, + 11, + 6, + 8, + 7, + 9, + 4, + 5, + 5, + 4, + 7, + 7, + 8, + 11, + 5, + 6, + 7, + 2, + 6, + 16, + 4, + 4, + 9, + 5, + 8, + 4, + 3, + 12, + 6, + 5, + 10, + 8, + 13, + 11, + 7, + 7, + 4, + 8, + 9, + 9, + 6, + 11, + 13, + 9, + 7, + 5, + 7, + 3, + 2, + 6, + 9, + 8, + 7, + 13, + 4, + 4, + 6, + 11, + 10, + 8, + 8, + 7, + 9, + 9, + 7, + 4, + 6, + 10, + 14, + 9, + 7, + 8, + 9, + 2, + 5, + 8, + 7, + 5, + 10, + 4, + 11, + 6, + 6, + 13, + 3, + 8, + 14, + 6, + 10, + 7, + 9, + 7, + 10, + 6, + 6, + 11, + 8, + 12, + 5, + 9, + 6, + 5, + 8, + 5, + 8, + 4, + 10, + 12, + 5, + 3, + 8, + 4, + 10, + 10, + 8, + 8, + 9, + 9, + 11, + 7, + 6, + 8, + 9, + 4, + 8, + 10, + 4, + 9, + 8, + 7, + 4, + 10, + 4, + 9, + 4, + 10, + 5, + 9, + 8, + 10, + 9, + 10, + 4, + 10, + 7, + 11, + 7, + 6, + 5, + 20, + 4, + 8, + 8, + 7, + 6, + 4, + 6, + 9, + 9, + 8, + 6, + 9, + 6, + 1, + 9, + 6, + 6, + 2, + 9, + 10, + 4, + 6, + 5, + 11, + 10, + 7, + 7, + 13, + 10, + 6, + 9, + 13, + 7, + 7, + 2, + 12, + 3, + 13, + 6, + 9, + 10, + 10, + 5, + 5, + 11, + 6, + 7, + 4, + 7, + 5, + 6, + 7, + 11, + 9, + 7, + 10, + 11, + 4, + 4, + 6, + 3, + 10, + 4, + 9, + 7, + 8, + 10, + 9, + 12, + 4, + 5, + 8, + 14, + 10, + 9, + 7, + 4, + 11, + 8, + 6, + 7, + 6, + 7, + 7, + 5, + 4, + 8, + 6, + 6, + 7, + 6, + 10, + 7, + 8, + 8, + 8, + 15, + 6, + 4, + 5, + 3, + 5, + 10, + 4, + 6, + 5, + 4, + 5, + 6, + 8, + 3, + 9, + 10, + 6, + 7, + 2, + 8, + 9, + 6, + 10, + 7, + 5, + 9, + 10, + 4, + 6, + 8, + 6, + 5, + 6, + 4, + 8, + 8, + 6, + 7, + 9, + 4, + 11, + 10, + 12, + 9, + 6, + 1, + 10, + 3, + 8, + 4, + 5, + 8, + 8, + 9, + 3, + 8, + 1, + 4, + 6, + 7, + 6, + 5, + 5, + 2, + 6, + 6, + 5, + 6, + 13, + 9, + 9, + 9, + 7, + 8, + 6, + 6, + 4, + 12, + 4, + 8, + 6, + 12, + 7, + 8, + 7, + 6, + 8, + 2, + 5, + 4, + 3, + 3, + 4, + 9, + 11, + 4, + 5, + 5, + 7, + 6, + 6, + 5, + 8, + 6, + 5, + 5, + 6, + 4, + 12, + 6, + 7, + 9, + 6, + 5, + 11, + 8, + 6, + 10, + 5, + 7, + 4, + 2, + 9, + 9, + 7, + 5, + 5, + 6, + 6, + 6, + 4, + 6, + 8, + 5, + 8, + 8, + 6, + 4, + 6, + 7, + 5, + 10, + 5, + 2, + 6, + 12, + 8, + 9, + 4, + 2, + 8, + 5, + 9, + 10, + 9, + 13, + 6, + 9, + 11, + 8, + 8, + 3, + 4, + 8, + 10, + 8, + 6, + 4, + 6, + 10, + 9, + 6, + 14, + 9, + 7, + 10, + 5, + 7, + 10, + 3, + 7, + 9, + 7, + 7, + 13, + 6, + 3, + 12, + 5, + 11, + 14, + 8, + 9, + 8, + 3, + 3, + 7, + 5, + 5, + 7, + 2, + 7, + 5, + 4, + 7, + 6, + 7, + 5, + 8, + 10, + 9, + 9, + 4, + 10, + 4, + 2, + 10, + 5, + 8, + 8, + 6, + 6, + 8, + 10, + 10, + 8, + 7, + 6, + 9, + 9, + 5, + 9, + 10, + 12, + 10, + 8, + 10, + 6, + 9, + 8, + 8, + 3, + 10, + 8, + 9, + 5, + 4, + 8, + 11, + 7, + 3, + 6, + 8, + 4, + 9, + 9, + 7, + 8, + 10, + 8, + 8, + 8, + 4, + 6, + 7, + 8, + 7, + 5, + 4, + 8, + 7, + 4, + 5, + 8, + 5, + 7, + 9, + 2, + 4, + 6, + 8, + 6, + 4, + 9, + 5, + 5, + 8, + 3, + 10, + 2, + 6, + 6, + 8, + 6, + 6, + 7, + 7, + 5, + 4, + 7, + 6, + 11, + 4, + 8, + 8, + 7, + 7, + 10, + 11, + 5, + 5, + 5, + 4, + 4, + 9, + 6, + 7, + 6, + 8, + 7, + 7, + 5, + 6, + 8, + 3, + 3, + 9, + 10, + 4, + 5, + 5, + 9, + 5, + 5, + 3, + 5, + 5, + 5, + 4, + 7, + 2, + 5, + 7, + 2, + 6, + 5, + 6, + 7, + 7, + 8, + 8, + 8, + 7, + 5, + 8, + 11, + 8, + 6, + 8, + 8, + 2, + 7, + 9, + 4, + 5, + 11, + 3, + 8, + 8, + 4, + 7, + 8, + 5, + 2, + 5, + 7, + 6, + 7, + 8, + 6, + 8, + 5, + 8, + 4, + 9, + 9, + 6, + 3, + 7, + 8, + 5, + 4, + 6, + 2, + 3, + 7, + 9, + 5, + 8, + 5, + 6, + 5, + 8, + 5, + 4, + 4, + 7, + 9, + 6, + 7, + 3, + 4, + 11, + 13, + 7, + 3, + 10, + 5, + 6, + 3, + 5, + 8, + 6, + 10, + 15, + 4, + 5, + 5, + 11, + 7, + 4, + 7, + 3, + 7, + 8, + 1, + 5, + 7, + 5, + 10, + 8, + 11, + 8, + 5, + 7, + 4, + 7, + 5, + 2, + 5, + 9, + 6, + 6, + 6, + 12, + 6, + 6, + 3, + 8, + 4, + 2, + 4, + 7, + 5, + 3, + 8, + 8, + 2, + 12, + 4, + 7, + 6, + 5, + 7, + 6, + 10, + 7, + 7, + 5, + 5, + 2, + 3, + 8, + 7, + 7, + 1, + 9, + 7, + 2, + 6, + 8, + 8, + 8, + 6, + 7, + 4, + 5, + 4, + 6, + 7, + 7, + 6, + 4, + 7, + 4, + 4, + 6, + 6, + 3, + 3, + 4, + 6, + 3, + 5, + 3, + 6, + 4, + 6, + 5, + 7, + 6, + 7, + 10, + 3, + 3, + 5, + 4, + 7, + 9, + 6, + 6, + 9, + 11, + 7, + 4, + 2, + 8, + 11, + 7, + 5, + 3, + 7, + 5, + 5, + 4, + 8, + 1, + 5, + 5, + 6, + 6, + 9, + 6, + 12, + 8, + 6, + 4, + 5, + 6, + 5, + 3, + 7, + 6, + 7, + 7, + 10, + 6, + 4, + 5, + 4, + 4, + 2, + 7, + 4, + 5, + 7, + 5, + 5, + 4, + 8, + 5, + 9, + 5, + 5, + 8, + 9, + 4, + 7, + 8, + 7, + 7, + 7, + 8, + 4, + 11, + 7, + 7, + 6, + 7, + 7, + 7, + 4, + 7, + 5, + 1, + 3, + 4, + 6, + 3, + 8, + 6, + 6, + 5, + 8, + 6, + 3, + 10, + 4, + 13, + 11, + 4, + 11, + 3, + 10, + 5, + 6, + 4, + 7, + 9, + 3, + 6, + 7, + 9, + 3, + 4, + 11, + 8, + 13, + 5, + 4, + 1, + 5, + 8, + 7, + 4, + 10, + 2, + 4, + 3, + 2, + 7, + 7, + 4, + 6, + 2, + 6, + 5, + 6, + 7, + 7, + 7, + 8, + 4, + 6, + 3, + 6, + 4, + 4, + 10, + 5, + 7, + 5, + 11, + 6, + 3, + 6, + 12, + 5, + 5, + 7, + 3, + 6, + 7, + 10, + 3, + 6, + 10, + 3, + 6, + 7, + 4, + 2, + 5, + 4, + 4, + 9, + 8, + 7, + 3, + 10, + 3, + 3, + 11, + 7, + 6, + 4, + 3, + 4, + 5, + 7, + 3, + 10, + 13, + 9, + 4, + 11, + 6, + 5, + 6, + 10, + 9, + 3, + 4, + 6, + 6, + 3, + 5, + 7, + 4, + 5, + 6, + 6, + 6, + 7, + 8, + 4, + 4, + 11, + 8, + 10, + 7, + 5, + 7, + 7, + 7, + 5, + 10, + 7, + 4, + 5, + 4, + 10, + 9, + 7, + 10, + 5, + 9, + 6, + 6, + 6, + 8, + 6, + 7, + 6, + 6, + 5, + 7, + 6, + 9, + 9, + 10, + 8, + 6, + 6, + 6, + 12, + 6, + 1, + 7, + 3, + 6, + 4, + 7, + 7, + 2, + 7, + 8, + 4, + 8, + 9, + 10, + 4, + 5, + 7, + 9, + 5, + 9, + 4, + 5, + 7, + 11, + 7, + 7, + 7, + 8, + 4, + 6, + 8, + 8, + 3, + 8, + 4, + 5, + 4, + 6, + 3, + 3, + 5, + 9, + 7, + 9, + 5, + 5, + 14, + 6, + 6, + 9, + 2, + 4, + 5, + 10, + 5, + 9, + 7, + 6, + 6, + 4, + 6, + 5, + 4, + 4, + 10, + 4, + 4, + 3, + 3, + 8, + 4, + 3, + 7, + 7, + 10, + 3, + 6, + 5, + 6, + 6, + 4, + 5, + 8, + 3, + 5, + 9, + 8, + 7, + 7, + 12, + 3, + 4, + 5, + 5, + 3, + 4, + 8, + 3, + 4, + 5, + 10, + 9, + 6, + 8, + 8, + 8, + 6, + 9, + 3, + 7, + 7, + 1, + 7, + 6, + 6, + 9, + 4, + 2, + 4, + 7, + 5, + 3, + 6, + 6, + 12, + 5, + 5, + 6, + 5, + 6, + 4, + 1, + 7, + 3, + 6, + 5, + 8, + 6, + 8, + 5, + 1, + 6, + 7, + 9, + 2, + 6, + 6, + 8, + 10, + 6, + 6, + 5, + 5, + 9, + 5, + 5, + 3, + 3, + 6, + 6, + 3, + 2, + 4, + 5, + 7, + 3, + 15, + 3, + 8, + 4, + 5, + 3, + 6, + 9, + 3, + 3, + 8, + 5, + 5, + 4, + 1, + 8, + 9, + 8, + 9, + 2, + 4, + 11, + 9, + 2, + 4, + 5, + 3, + 6, + 9, + 5, + 7, + 2, + 10, + 9, + 13, + 6, + 8, + 4, + 10, + 7, + 5, + 5, + 8, + 4, + 6, + 6, + 8, + 6, + 8, + 7, + 5, + 5, + 4, + 4, + 4, + 6, + 9, + 7, + 7, + 6, + 9, + 3, + 1, + 4, + 4, + 7, + 7, + 4, + 7, + 7, + 4, + 6, + 4, + 7, + 6, + 8, + 5, + 6, + 9, + 3, + 5, + 8, + 3, + 7, + 4, + 8, + 7, + 2, + 5, + 7, + 5, + 3, + 5, + 7, + 3, + 2, + 5, + 5, + 6, + 3, + 6, + 1, + 4, + 9, + 5, + 5, + 6, + 7, + 6, + 6, + 5, + 6, + 8, + 8, + 11, + 5, + 5, + 5, + 9, + 5, + 6, + 11, + 2, + 10, + 8, + 5, + 6, + 7, + 7, + 5, + 8, + 3, + 4, + 3, + 5, + 8, + 5, + 3, + 6, + 7, + 8, + 7, + 2, + 4, + 6, + 13, + 5, + 4, + 7, + 8, + 6, + 4, + 6, + 7, + 1, + 3, + 4, + 4, + 7, + 5, + 5, + 7, + 5, + 5, + 7, + 3, + 4, + 2, + 5, + 3, + 3, + 8, + 8, + 7, + 6, + 8, + 5, + 5, + 3, + 7, + 7, + 6, + 4, + 6, + 4, + 8, + 2, + 7, + 7, + 5, + 3, + 5, + 6, + 5, + 6, + 3, + 7, + 6, + 7, + 4, + 10, + 2, + 7, + 4, + 10, + 3, + 5, + 12, + 6, + 6, + 3, + 4, + 5, + 4, + 5, + 6, + 4, + 9, + 7, + 3, + 4, + 2, + 6, + 4, + 4, + 2, + 4, + 4, + 7, + 9, + 6, + 1, + 9, + 4, + 5, + 10, + 3, + 8, + 8, + 5, + 3, + 4, + 4, + 5, + 3, + 2, + 9, + 6, + 5, + 4, + 6, + 9, + 3, + 9, + 6, + 5, + 1, + 8, + 7, + 10, + 4, + 7, + 6, + 5, + 3, + 5, + 4, + 5, + 2, + 3, + 5, + 9, + 5, + 7, + 3, + 10, + 6, + 9, + 5, + 6, + 3, + 5, + 4, + 4, + 7, + 7, + 5, + 5, + 4, + 7, + 5, + 5, + 2, + 6, + 4, + 6, + 8, + 5, + 8, + 5, + 6, + 2, + 5, + 3, + 8, + 7, + 8, + 1, + 4, + 3, + 6, + 3, + 7, + 8, + 2, + 5, + 7, + 5, + 5, + 2, + 4, + 4, + 7, + 6, + 5, + 4, + 7, + 5, + 5, + 6, + 8, + 5, + 6, + 4, + 5, + 6, + 6, + 6, + 3, + 2, + 7, + 1, + 4, + 3, + 3, + 3, + 7, + 7, + 4, + 6, + 6, + 8, + 6, + 5, + 4, + 9, + 7, + 1, + 10, + 6, + 7, + 6, + 6, + 4, + 5, + 2, + 4, + 3, + 5, + 4, + 9, + 5, + 5, + 7, + 7, + 2, + 2, + 6, + 6, + 3, + 7, + 4, + 5, + 12, + 8, + 9, + 2, + 23, + 7, + 2, + 5, + 6, + 3, + 6, + 6, + 6, + 10, + 8, + 2, + 3, + 10, + 2, + 2, + 5, + 4, + 3, + 5, + 5, + 6, + 5, + 4, + 8, + 6, + 3, + 8, + 3, + 9, + 5, + 7, + 4, + 1, + 5, + 7, + 3, + 5, + 7, + 8, + 2, + 5, + 7, + 3, + 5, + 2, + 2, + 7, + 12, + 8, + 3, + 4, + 5, + 4, + 7, + 8, + 2, + 3, + 9, + 4, + 3, + 4, + 4, + 4, + 7, + 12, + 5, + 3, + 8, + 5, + 5, + 4, + 1, + 1, + 5, + 5, + 6, + 1, + 6, + 6, + 3, + 2, + 2, + 6, + 7, + 3, + 4, + 3, + 7, + 7, + 7, + 3, + 6, + 3, + 5, + 7, + 3, + 6, + 8, + 8, + 4, + 3, + 3, + 5, + 11, + 7, + 5, + 6, + 5, + 8, + 7, + 8, + 1, + 6, + 4, + 4, + 12, + 6, + 8, + 3, + 8, + 4, + 5, + 7, + 2, + 4, + 7, + 2, + 4, + 12, + 9, + 3, + 2, + 4, + 5, + 3, + 6, + 5, + 5, + 6, + 1, + 1, + 2, + 3, + 3, + 7, + 6, + 1, + 9, + 4, + 4, + 2, + 8, + 5, + 8, + 1, + 2, + 9, + 9, + 5, + 7, + 6, + 11, + 4, + 6, + 5, + 4, + 6, + 10, + 5, + 6, + 5, + 6, + 7, + 6, + 4, + 7, + 3, + 3, + 4, + 3, + 6, + 5, + 3, + 1, + 3, + 2, + 9, + 3, + 3, + 5, + 4, + 1, + 5, + 1, + 4, + 6, + 6, + 1, + 5, + 3, + 4, + 4, + 11, + 5, + 4, + 4, + 6, + 4, + 9, + 10, + 3, + 5, + 2, + 7, + 2, + 6, + 4, + 2, + 10, + 4, + 7, + 9, + 2, + 7, + 7, + 4, + 4, + 4, + 4, + 3, + 5, + 7, + 12, + 2, + 3, + 7, + 7, + 4, + 4, + 8, + 5, + 7, + 4, + 4, + 4, + 2, + 6, + 6, + 10, + 5, + 8, + 4, + 8, + 7, + 7, + 4, + 4, + 4, + 5, + 3, + 4, + 4, + 3, + 5, + 3, + 5, + 1, + 6, + 8, + 3, + 7, + 5, + 5, + 5, + 8, + 6, + 2, + 9, + 4, + 6, + 4, + 3, + 9, + 13, + 7, + 3, + 8, + 14, + 1, + 10, + 7, + 7, + 4, + 5, + 1, + 6, + 3, + 1, + 5, + 3, + 5, + 7, + 3, + 4, + 3, + 12, + 4, + 2, + 3, + 1, + 5, + 8, + 4, + 5, + 2, + 4, + 5, + 5, + 5, + 11, + 7, + 6, + 5, + 5, + 8, + 5, + 5, + 4, + 1, + 3, + 6, + 6, + 2, + 8, + 4, + 6, + 3, + 5, + 5, + 5, + 5, + 4, + 1, + 3, + 8, + 4, + 4, + 4, + 5, + 6, + 4, + 5, + 4, + 4, + 2, + 5, + 4, + 5, + 3, + 5, + 3, + 4, + 3, + 6, + 1, + 6, + 4, + 2, + 2, + 4, + 2, + 3, + 1, + 8, + 6, + 2, + 5, + 4, + 4, + 6, + 5, + 2, + 7, + 2, + 3, + 5, + 5, + 5, + 3, + 7, + 8, + 5, + 5, + 3, + 4, + 5, + 2, + 5, + 3, + 4, + 5, + 4, + 1, + 8, + 7, + 6, + 6, + 6, + 2, + 3, + 9, + 3, + 8, + 8, + 4, + 8, + 4, + 2, + 4, + 6, + 3, + 4, + 9, + 6, + 1, + 4, + 4, + 7, + 5, + 5, + 5, + 3, + 5, + 4, + 3, + 4, + 5, + 4, + 1, + 5, + 3, + 6, + 7, + 2, + 7, + 6, + 8, + 6, + 3, + 8, + 2, + 5, + 7, + 5, + 7, + 6, + 2, + 6, + 5, + 8, + 5, + 7, + 6, + 5, + 3, + 4, + 5, + 2, + 4, + 3, + 7, + 5, + 2, + 7, + 1, + 5, + 5, + 7, + 3, + 5, + 4, + 3, + 6, + 7, + 2, + 5, + 2, + 2, + 2, + 1, + 7, + 3, + 4, + 3, + 2, + 5, + 6, + 2, + 3, + 2, + 7, + 3, + 4, + 3, + 2, + 5, + 5, + 4, + 2, + 5, + 3, + 5, + 4, + 9, + 3, + 4, + 2, + 2, + 6, + 7, + 3, + 4, + 3, + 7, + 5, + 8, + 3, + 5, + 6, + 2, + 3, + 5, + 3, + 9, + 5, + 6, + 2, + 3, + 4, + 5, + 9, + 4, + 4, + 1, + 1, + 4, + 7, + 3, + 1, + 2, + 7, + 7, + 4, + 3, + 6, + 7, + 5, + 9, + 2, + 5, + 8, + 5, + 2, + 3, + 4, + 5, + 5, + 5, + 4, + 4, + 4, + 3, + 2, + 3, + 3, + 1, + 7, + 3, + 6, + 7, + 4, + 4, + 11, + 5, + 5, + 7, + 9, + 1, + 6, + 4, + 3, + 6, + 6, + 4, + 1, + 4, + 5, + 6, + 9, + 7, + 4, + 5, + 4, + 3, + 4, + 5, + 4, + 3, + 7, + 8, + 6, + 3, + 6, + 6, + 8, + 3, + 4, + 3, + 3, + 4, + 3, + 9, + 6, + 5, + 8, + 5, + 5, + 8, + 4, + 5, + 3, + 4, + 2, + 7, + 3, + 3, + 5, + 7, + 7, + 4, + 2, + 3, + 9, + 5, + 6, + 3, + 4, + 11, + 7, + 8, + 9, + 2, + 11, + 7, + 5, + 4, + 5, + 3, + 3, + 4, + 8, + 4, + 6, + 8, + 4, + 1, + 3, + 4, + 5, + 2, + 7, + 6, + 5, + 7, + 3, + 2, + 1, + 2, + 3, + 3, + 1, + 6, + 9, + 5, + 5, + 5, + 5, + 7, + 4, + 4, + 3, + 7, + 1, + 5, + 3, + 4, + 6, + 7, + 3, + 1, + 8, + 1, + 6, + 5, + 7, + 3, + 1, + 6, + 2, + 4, + 5, + 4, + 5, + 5, + 5, + 10, + 4, + 4, + 6, + 4, + 5, + 5, + 1, + 1, + 3, + 7, + 3, + 3, + 5, + 5, + 4, + 2, + 2, + 2, + 7, + 3, + 8, + 2, + 7, + 6, + 5, + 2, + 7, + 2, + 1, + 1, + 2, + 7, + 6, + 4, + 2, + 3, + 3, + 3, + 8, + 2, + 4, + 1, + 3, + 3, + 4, + 6, + 4, + 6, + 1, + 8, + 3, + 7, + 4, + 3, + 4, + 1, + 3, + 4, + 9, + 1, + 4, + 7, + 9, + 3, + 4, + 7, + 3, + 5, + 5, + 6, + 3, + 7, + 1, + 3, + 5, + 4, + 5, + 6, + 3, + 5, + 6, + 4, + 5, + 4, + 4, + 4, + 5, + 7, + 8, + 2, + 3, + 7, + 5, + 5, + 7, + 3, + 8, + 6, + 3, + 8, + 1, + 1, + 1, + 2, + 3, + 5, + 4, + 5, + 5, + 5, + 5, + 2, + 5, + 2, + 5, + 4, + 7, + 4, + 4, + 5, + 3, + 4, + 2, + 5, + 4, + 6, + 3, + 5, + 7, + 1, + 5, + 6, + 6, + 5, + 2, + 1, + 8, + 5, + 3, + 4, + 7, + 1, + 3, + 3, + 2, + 6, + 4, + 5, + 5, + 3, + 2, + 5, + 7, + 5, + 4, + 5, + 2, + 7, + 4, + 5, + 8, + 6, + 4, + 2, + 5, + 3, + 3, + 4, + 6, + 3, + 7, + 5, + 4, + 2, + 7, + 1, + 3, + 2, + 4, + 3, + 3, + 3, + 5, + 4, + 4, + 4, + 6, + 5, + 3, + 6, + 11, + 5, + 7, + 8, + 4, + 2, + 7, + 7, + 2, + 3, + 5, + 5, + 6, + 2, + 3, + 4, + 3, + 2, + 3, + 6, + 4, + 6, + 6, + 2, + 7, + 2, + 5, + 9, + 3, + 7, + 3, + 5, + 6, + 9, + 5, + 4, + 4, + 7, + 7, + 5, + 5, + 8, + 4, + 5, + 5, + 5, + 2, + 2, + 5, + 5, + 1, + 5, + 6, + 4, + 6, + 6, + 8, + 3, + 6, + 4, + 6, + 7, + 6, + 3, + 5, + 2, + 5, + 3, + 5, + 2, + 3, + 5, + 5, + 4, + 7, + 5, + 6, + 3, + 8, + 1, + 4, + 5, + 5, + 1, + 2, + 3, + 4, + 6, + 5, + 1, + 4, + 4, + 2, + 2, + 8, + 5, + 3, + 2, + 4, + 6, + 5, + 1, + 4, + 2, + 8, + 6, + 5, + 3, + 4, + 2, + 7, + 4, + 4, + 7, + 2, + 6, + 6, + 2, + 4, + 8, + 4, + 4, + 3, + 3, + 5, + 5, + 3, + 3, + 2, + 5, + 2, + 6, + 4, + 5, + 3, + 2, + 1, + 7, + 1, + 3, + 4, + 4, + 1, + 6, + 6, + 6, + 6, + 1, + 6, + 5, + 4, + 5, + 7, + 2, + 4, + 5, + 5, + 5, + 4, + 1, + 3, + 3, + 3, + 4, + 2, + 1, + 2, + 4, + 6, + 7, + 2, + 4, + 2, + 5, + 2, + 4, + 2, + 6, + 3, + 2, + 1, + 4, + 6, + 7, + 8, + 7, + 2, + 3, + 4, + 2, + 4, + 5, + 7, + 4, + 7, + 2, + 6, + 6, + 5, + 5, + 6, + 4, + 4, + 5, + 5, + 2, + 2, + 4, + 4, + 3, + 4, + 2, + 4, + 2, + 5, + 6, + 3, + 6, + 4, + 3, + 6, + 2, + 6, + 2, + 7, + 4, + 10, + 2, + 5, + 3, + 2, + 5, + 4, + 4, + 7, + 2, + 2, + 5, + 5, + 2, + 5, + 2, + 3, + 3, + 4, + 5, + 3, + 6, + 6, + 8, + 4, + 3, + 9, + 7, + 3, + 5, + 3, + 5, + 4, + 5, + 2, + 6, + 6, + 1, + 2, + 5, + 8, + 3, + 2, + 5, + 7, + 4, + 5, + 5, + 3, + 5, + 3, + 2, + 6, + 5, + 4, + 4, + 2, + 4, + 4, + 5, + 4, + 6, + 3, + 3, + 5, + 2, + 1, + 1, + 8, + 6, + 1, + 5, + 1, + 5, + 1, + 4, + 4, + 1, + 2, + 5, + 5, + 2, + 2, + 5, + 7, + 1, + 7, + 8, + 1, + 2, + 4, + 2, + 5, + 4, + 6, + 4, + 1, + 5, + 3, + 2, + 2, + 4, + 3, + 7, + 5, + 2, + 8, + 1, + 6, + 6, + 1, + 4, + 6, + 4, + 3, + 2, + 5, + 7, + 5, + 4, + 5, + 2, + 5, + 4, + 2, + 3, + 7, + 2, + 5, + 5, + 5, + 5, + 6, + 4, + 3, + 5, + 7, + 2, + 5, + 1, + 2, + 6, + 4, + 1, + 4, + 2, + 3, + 2, + 1, + 9, + 2, + 2, + 8, + 3, + 5, + 2, + 4, + 8, + 1, + 4, + 2, + 6, + 2, + 3, + 5, + 1, + 3, + 3, + 8, + 6, + 2, + 4, + 1, + 3, + 5, + 6, + 5, + 5, + 2, + 4, + 4, + 12, + 3, + 5, + 3, + 4, + 3, + 4, + 9, + 6, + 5, + 5, + 4, + 6, + 6, + 6, + 4, + 6, + 4, + 6, + 2, + 4, + 6, + 2, + 2, + 1, + 3, + 7, + 4, + 3, + 3, + 6, + 3, + 2, + 5, + 4, + 4, + 6, + 4, + 3, + 6, + 3, + 3, + 7, + 6, + 2, + 1, + 2, + 4, + 2, + 4, + 4, + 4, + 3, + 3, + 7, + 5, + 3, + 3, + 6, + 4, + 6, + 3, + 3, + 3, + 7, + 6, + 4, + 3, + 2, + 3, + 1, + 5, + 3, + 3, + 4, + 2, + 6, + 3, + 3, + 3, + 4, + 5, + 7, + 4, + 1, + 5, + 8, + 6, + 5, + 4, + 1, + 4, + 2, + 7, + 3, + 5, + 3, + 3, + 3, + 4, + 4, + 4, + 7, + 2, + 4, + 5, + 7, + 9, + 3, + 1, + 7, + 4, + 4, + 3, + 7, + 3, + 7, + 3, + 2, + 2, + 9, + 2, + 4, + 4, + 5, + 4, + 5, + 4, + 4, + 5, + 4, + 1, + 3, + 5, + 5, + 3, + 1, + 3, + 6, + 4, + 3, + 5, + 1, + 2, + 2, + 3, + 6, + 4, + 3, + 3, + 2, + 4, + 6, + 3, + 6, + 7, + 2, + 7, + 2, + 1, + 4, + 4, + 5, + 7, + 2, + 4, + 6, + 2, + 5, + 3, + 2, + 2, + 1, + 5, + 4, + 3, + 2, + 4, + 2, + 4, + 6, + 3, + 8, + 2, + 7, + 3, + 6, + 1, + 1, + 4, + 3, + 5, + 8, + 4, + 2, + 3, + 4, + 2, + 4, + 1, + 6, + 5, + 2, + 4, + 2, + 3, + 7, + 5, + 7, + 7, + 5, + 2, + 5, + 1, + 4, + 4, + 3, + 3, + 4, + 4, + 5, + 7, + 3, + 3, + 4, + 2, + 4, + 4, + 5, + 3, + 1, + 5, + 6, + 7, + 3, + 2, + 6, + 5, + 4, + 2, + 3, + 3, + 6, + 4, + 3, + 6, + 3, + 1, + 7, + 3, + 4, + 5, + 4, + 4, + 5, + 2, + 4, + 5, + 5, + 4, + 8, + 2, + 1, + 5, + 3, + 6, + 3, + 2, + 5, + 1, + 3, + 3, + 4, + 3, + 2, + 6, + 2, + 4, + 4, + 5, + 5, + 3, + 2, + 5, + 9, + 1, + 3, + 9, + 5, + 1, + 6, + 4, + 2, + 3, + 7, + 3, + 2, + 4, + 5, + 1, + 4, + 5, + 11, + 4, + 2, + 3, + 4, + 4, + 5, + 3, + 4, + 2, + 5, + 2, + 1, + 6, + 5, + 5, + 2, + 3, + 3, + 2, + 7, + 2, + 1, + 4, + 3, + 2, + 2, + 3, + 6, + 5, + 3, + 3, + 2, + 3, + 2, + 5, + 7, + 3, + 2, + 2, + 6, + 6, + 1, + 7, + 1, + 7, + 2, + 4, + 5, + 4, + 3, + 7, + 3, + 8, + 3, + 7, + 5, + 3, + 2, + 3, + 1, + 3, + 1, + 6, + 4, + 4, + 3, + 1, + 5, + 4, + 6, + 7, + 3, + 1, + 3, + 2, + 5, + 4, + 4, + 4, + 3, + 7, + 3, + 3, + 1, + 6, + 8, + 4, + 3, + 1, + 6, + 4, + 4, + 2, + 3, + 4, + 4, + 4, + 3, + 5, + 3, + 3, + 5, + 7, + 5, + 5, + 2, + 2, + 1, + 5, + 5, + 4, + 4, + 3, + 3, + 3, + 1, + 4, + 4, + 3, + 4, + 2, + 6, + 6, + 3, + 1, + 1, + 5, + 2, + 3, + 2, + 5, + 3, + 4, + 2, + 3, + 2, + 3, + 5, + 3, + 5, + 2, + 6, + 3, + 2, + 3, + 2, + 2, + 4, + 6, + 2, + 6, + 3, + 1, + 5, + 2, + 3, + 8, + 8, + 1, + 2, + 4, + 3, + 4, + 3, + 4, + 2, + 5, + 3, + 2, + 6, + 5, + 2, + 4, + 4, + 4, + 3, + 4, + 3, + 4, + 5, + 3, + 3, + 3, + 3, + 4, + 2, + 4, + 6, + 1, + 3, + 4, + 4, + 4, + 8, + 2, + 3, + 3, + 5, + 2, + 3, + 2, + 1, + 5, + 6, + 4, + 3, + 8, + 3, + 3, + 6, + 4, + 1, + 3, + 3, + 4, + 3, + 4, + 4, + 5, + 5, + 5, + 3, + 1, + 4, + 6, + 4, + 4, + 7, + 3, + 3, + 2, + 7, + 2, + 4, + 5, + 7, + 3, + 4, + 3, + 4, + 7, + 5, + 9, + 4, + 1, + 1, + 3, + 3, + 4, + 2, + 5, + 6, + 2, + 5, + 4, + 5, + 6, + 4, + 1, + 4, + 4, + 5, + 3, + 6, + 4, + 3, + 6, + 3, + 5, + 4, + 1, + 5, + 2, + 4, + 2, + 1, + 5, + 2, + 3, + 2, + 3, + 3, + 3, + 6, + 2, + 3, + 2, + 5, + 5, + 4, + 6, + 3, + 1, + 4, + 5, + 8, + 5, + 2, + 5, + 4, + 3, + 2, + 3, + 7, + 3, + 3, + 2, + 5, + 3, + 1, + 5, + 3, + 3, + 1, + 3, + 4, + 5, + 6, + 4, + 5, + 2, + 4, + 4, + 4, + 4, + 5, + 5, + 2, + 5, + 4, + 4, + 4, + 1, + 2, + 2, + 3, + 3, + 4, + 5, + 6, + 5, + 3, + 2, + 4, + 2, + 3, + 1, + 6, + 5, + 3, + 3, + 7, + 6, + 5, + 1, + 4, + 2, + 4, + 8, + 1, + 4, + 1, + 6, + 3, + 6, + 5, + 3, + 3, + 2, + 6, + 2, + 3, + 6, + 3, + 6, + 2, + 5, + 3, + 5, + 1, + 8, + 6, + 3, + 3, + 3, + 3, + 1, + 2, + 3, + 8, + 4, + 1, + 1, + 5, + 3, + 4, + 3, + 7, + 4, + 3, + 2, + 5, + 4, + 7, + 7, + 7, + 5, + 3, + 4, + 7, + 3, + 4, + 6, + 2, + 8, + 2, + 5, + 6, + 3, + 2, + 4, + 3, + 3, + 6, + 2, + 3, + 1, + 3, + 4, + 5, + 2, + 4, + 1, + 2, + 4, + 4, + 4, + 6, + 4, + 3, + 2, + 4, + 3, + 6, + 2, + 3, + 6, + 1, + 3, + 5, + 6, + 2, + 6, + 4, + 3, + 4, + 1, + 4, + 3, + 1, + 2, + 2, + 4, + 2, + 3, + 4, + 3, + 5, + 5, + 1, + 4, + 2, + 3, + 1, + 4, + 5, + 5, + 4, + 5, + 4, + 4, + 2, + 3, + 2, + 1, + 5, + 5, + 6, + 4, + 2, + 3, + 4, + 2, + 1, + 4, + 3, + 1, + 9, + 3, + 5, + 1, + 2, + 2, + 6, + 2, + 1, + 3, + 1, + 4, + 2, + 5, + 2, + 3, + 10, + 1, + 3, + 8, + 1, + 4, + 4, + 2, + 1, + 6, + 2, + 4, + 3, + 4, + 1, + 2, + 2, + 7, + 5, + 1, + 6, + 3, + 1, + 10, + 5, + 3, + 2, + 2, + 1, + 2, + 4, + 1, + 5, + 8, + 1, + 3, + 5, + 2, + 7, + 5, + 5, + 2, + 5, + 1, + 1, + 3, + 9, + 2, + 3, + 3, + 1, + 2, + 1, + 2, + 4, + 6, + 4, + 6, + 3, + 3, + 5, + 6, + 3, + 3, + 3, + 6, + 2, + 4, + 4, + 3, + 2, + 8, + 3, + 5, + 5, + 2, + 3, + 2, + 2, + 3, + 6, + 10, + 1, + 3, + 5, + 5, + 1, + 1, + 3, + 4, + 4, + 2, + 2, + 6, + 2, + 4, + 7, + 6, + 2, + 4, + 2, + 5, + 2, + 3, + 2, + 3, + 1, + 3, + 4, + 3, + 2, + 8, + 5, + 1, + 3, + 4, + 2, + 6, + 6, + 3, + 4, + 3, + 4, + 3, + 3, + 3, + 3, + 1, + 2, + 5, + 8, + 3, + 1, + 1, + 6, + 3, + 3, + 5, + 4, + 2, + 4, + 2, + 6, + 1, + 4, + 4, + 2, + 4, + 4, + 6, + 7, + 7, + 4, + 2, + 3, + 7, + 2, + 5, + 1, + 3, + 3, + 3, + 3, + 7, + 1, + 3, + 3, + 7, + 1, + 2, + 4, + 3, + 2, + 2, + 7, + 7, + 10, + 4, + 2, + 4, + 7, + 3, + 6, + 3, + 3, + 4, + 8, + 1, + 4, + 3, + 2, + 2, + 2, + 4, + 4, + 4, + 1, + 3, + 7, + 1, + 2, + 2, + 2, + 2, + 5, + 4, + 4, + 5, + 1, + 2, + 4, + 5, + 4, + 2, + 5, + 6, + 2, + 7, + 3, + 2, + 3, + 2, + 1, + 4, + 3, + 7, + 3, + 8, + 2, + 3, + 3, + 3, + 3, + 8, + 5, + 6, + 6, + 1, + 1, + 2, + 1, + 1, + 1, + 5, + 1, + 3, + 2, + 5, + 5, + 3, + 4, + 5, + 4, + 4, + 2, + 4, + 2, + 6, + 1, + 2, + 2, + 4, + 3, + 2, + 3, + 4, + 2, + 2, + 3, + 3, + 4, + 1, + 7, + 2, + 1, + 6, + 7, + 1, + 2, + 6, + 4, + 2, + 8, + 3, + 3, + 1, + 2, + 3, + 5, + 5, + 4, + 3, + 3, + 1, + 4, + 3, + 5, + 6, + 1, + 4, + 2, + 3, + 3, + 2, + 2, + 3, + 3, + 1, + 2, + 7, + 3, + 3, + 4, + 1, + 5, + 4, + 1, + 7, + 3, + 1, + 1, + 3, + 6, + 2, + 3, + 2, + 3, + 2, + 2, + 7, + 3, + 9, + 5, + 4, + 3, + 2, + 2, + 1, + 1, + 5, + 4, + 2, + 2, + 8, + 5, + 2, + 6, + 4, + 4, + 2, + 3, + 4, + 2, + 2, + 2, + 3, + 1, + 1, + 3, + 4, + 2, + 6, + 6, + 6, + 2, + 2, + 1, + 5, + 1, + 4, + 3, + 4, + 5, + 3, + 3, + 2, + 3, + 3, + 8, + 2, + 3, + 4, + 2, + 6, + 8, + 2, + 4, + 1, + 3, + 3, + 2, + 3, + 2, + 5, + 5, + 3, + 4, + 4, + 8, + 3, + 7, + 1, + 1, + 4, + 2, + 4, + 1, + 4, + 5, + 2, + 2, + 6, + 5, + 2, + 1, + 2, + 2, + 3, + 3, + 1, + 2, + 4, + 5, + 3, + 1, + 4, + 3, + 7, + 3, + 5, + 5, + 2, + 2, + 1, + 1, + 6, + 1, + 5, + 2, + 5, + 3, + 4, + 2, + 2, + 2, + 1, + 2, + 7, + 2, + 3, + 4, + 3, + 4, + 5, + 2, + 2, + 3, + 3, + 3, + 4, + 6, + 3, + 2, + 4, + 2, + 2, + 7, + 18, + 2, + 2, + 6, + 3, + 4, + 4, + 5, + 5, + 4, + 2, + 3, + 6, + 4, + 3, + 3, + 1, + 6, + 4, + 4, + 2, + 3, + 1, + 7, + 4, + 4, + 1, + 1, + 3, + 3, + 6, + 6, + 2, + 1, + 2, + 3, + 3, + 3, + 3, + 3, + 5, + 5, + 1, + 6, + 7, + 6, + 2, + 4, + 2, + 4, + 2, + 3, + 8, + 3, + 2, + 4, + 4, + 4, + 9, + 1, + 5, + 2, + 2, + 4, + 2, + 1, + 3, + 2, + 3, + 5, + 2, + 2, + 1, + 3, + 3, + 2, + 4, + 4, + 3, + 1, + 3, + 1, + 4, + 1, + 4, + 2, + 6, + 2, + 1, + 2, + 2, + 5, + 3, + 4, + 6, + 2, + 1, + 3, + 4, + 1, + 3, + 1, + 3, + 1, + 2, + 2, + 4, + 2, + 1, + 2, + 5, + 3, + 7, + 3, + 3, + 4, + 5, + 1, + 2, + 3, + 2, + 4, + 4, + 2, + 3, + 4, + 1, + 4, + 7, + 8, + 3, + 7, + 2, + 9, + 2, + 6, + 3, + 3, + 4, + 3, + 1, + 1, + 5, + 4, + 1, + 1, + 3, + 4, + 3, + 3, + 6, + 3, + 3, + 2, + 3, + 2, + 1, + 2, + 2, + 5, + 3, + 1, + 6, + 7, + 1, + 2, + 1, + 5, + 5, + 4, + 1, + 4, + 1, + 4, + 4, + 4, + 1, + 4, + 4, + 3, + 3, + 3, + 2, + 4, + 3, + 2, + 4, + 3, + 1, + 1, + 5, + 7, + 3, + 1, + 4, + 5, + 3, + 4, + 2, + 3, + 4, + 5, + 2, + 2, + 3, + 1, + 4, + 1, + 1, + 1, + 3, + 5, + 2, + 1, + 2, + 5, + 4, + 4, + 3, + 1, + 1, + 2, + 3, + 3, + 4, + 1, + 2, + 6, + 1, + 3, + 1, + 2, + 3, + 6, + 4, + 4, + 1, + 2, + 1, + 1, + 6, + 2, + 1, + 5, + 3, + 2, + 2, + 1, + 4, + 4, + 1, + 1, + 3, + 6, + 1, + 1, + 1, + 6, + 2, + 3, + 2, + 1, + 5, + 1, + 5, + 5, + 5, + 1, + 4, + 3, + 1, + 2, + 7, + 3, + 1, + 3, + 1, + 2, + 2, + 6, + 3, + 4, + 4, + 6, + 1, + 4, + 4, + 7, + 4, + 6, + 3, + 3, + 5, + 4, + 3, + 4, + 2, + 3, + 2, + 5, + 1, + 5, + 2, + 3, + 3, + 6, + 3, + 1, + 2, + 4, + 4, + 2, + 2, + 7, + 7, + 8, + 2, + 2, + 3, + 2, + 4, + 1, + 4, + 1, + 2, + 1, + 3, + 3, + 1, + 3, + 9, + 2, + 1, + 2, + 6, + 2, + 3, + 6, + 1, + 4, + 2, + 3, + 2, + 3, + 6, + 4, + 1, + 1, + 1, + 4, + 3, + 2, + 1, + 3, + 3, + 3, + 7, + 4, + 6, + 3, + 2, + 4, + 1, + 6, + 4, + 2, + 3, + 1, + 2, + 3, + 3, + 4, + 7, + 4, + 3, + 1, + 2, + 4, + 4, + 2, + 4, + 1, + 3, + 4, + 5, + 3, + 3, + 1, + 6, + 1, + 5, + 3, + 2, + 5, + 4, + 5, + 7, + 1, + 4, + 2, + 5, + 3, + 1, + 6, + 2, + 4, + 2, + 3, + 4, + 3, + 3, + 2, + 1, + 3, + 1, + 2, + 2, + 2, + 2, + 2, + 1, + 3, + 1, + 1, + 2, + 5, + 4, + 4, + 5, + 3, + 3, + 3, + 2, + 3, + 2, + 3, + 4, + 1, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 6, + 2, + 2, + 3, + 3, + 2, + 3, + 4, + 2, + 6, + 3, + 4, + 2, + 4, + 2, + 5, + 4, + 1, + 5, + 3, + 2, + 4, + 1, + 3, + 1, + 5, + 4, + 2, + 2, + 3, + 4, + 2, + 7, + 4, + 3, + 2, + 6, + 4, + 5, + 2, + 1, + 5, + 3, + 1, + 4, + 4, + 2, + 3, + 2, + 4, + 5, + 3, + 4, + 2, + 7, + 2, + 2, + 5, + 3, + 2, + 3, + 3, + 4, + 2, + 2, + 1, + 2, + 4, + 1, + 3, + 2, + 1, + 3, + 1, + 4, + 3, + 1, + 8, + 3, + 3, + 2, + 2, + 1, + 3, + 4, + 3, + 6, + 4, + 3, + 2, + 3, + 2, + 4, + 3, + 3, + 3, + 3, + 3, + 2, + 2, + 3, + 5, + 4, + 1, + 4, + 4, + 3, + 3, + 6, + 1, + 5, + 1, + 3, + 1, + 4, + 2, + 4, + 1, + 1, + 1, + 4, + 3, + 2, + 4, + 5, + 5, + 1, + 3, + 1, + 3, + 3, + 3, + 4, + 3, + 5, + 2, + 2, + 4, + 4, + 2, + 3, + 3, + 2, + 5, + 2, + 2, + 2, + 6, + 2, + 1, + 1, + 1, + 3, + 2, + 3, + 4, + 4, + 2, + 3, + 1, + 3, + 6, + 3, + 4, + 5, + 3, + 3, + 1, + 2, + 3, + 2, + 2, + 2, + 4, + 4, + 4, + 5, + 2, + 2, + 1, + 3, + 2, + 2, + 3, + 4, + 1, + 2, + 2, + 4, + 4, + 3, + 5, + 6, + 2, + 3, + 3, + 3, + 2, + 5, + 3, + 5, + 3, + 4, + 7, + 4, + 5, + 5, + 2, + 4, + 5, + 3, + 2, + 1, + 3, + 4, + 4, + 4, + 1, + 3, + 3, + 4, + 3, + 6, + 4, + 3, + 8, + 4, + 3, + 1, + 3, + 2, + 3, + 4, + 2, + 2, + 4, + 2, + 1, + 2, + 4, + 3, + 1, + 3, + 3, + 3, + 1, + 2, + 3, + 8, + 7, + 6, + 2, + 2, + 4, + 2, + 1, + 5, + 3, + 4, + 3, + 5, + 1, + 4, + 3, + 7, + 1, + 5, + 2, + 2, + 3, + 4, + 4, + 6, + 3, + 2, + 1, + 5, + 1, + 2, + 4, + 2, + 2, + 3, + 3, + 2, + 5, + 4, + 1, + 4, + 1, + 3, + 2, + 3, + 2, + 2, + 4, + 4, + 4, + 3, + 4, + 3, + 1, + 4, + 5, + 2, + 1, + 2, + 3, + 2, + 3, + 1, + 8, + 1, + 3, + 3, + 4, + 5, + 2, + 4, + 3, + 1, + 5, + 2, + 1, + 3, + 6, + 1, + 5, + 2, + 5, + 2, + 1, + 1, + 3, + 3, + 4, + 4, + 1, + 4, + 1, + 5, + 3, + 4, + 2, + 2, + 5, + 2, + 2, + 3, + 3, + 3, + 2, + 2, + 4, + 4, + 2, + 3, + 3, + 3, + 5, + 2, + 4, + 3, + 3, + 5, + 1, + 1, + 4, + 7, + 3, + 4, + 2, + 1, + 1, + 4, + 2, + 1, + 2, + 6, + 1, + 3, + 3, + 2, + 3, + 3, + 2, + 5, + 2, + 3, + 3, + 2, + 7, + 5, + 6, + 5, + 2, + 3, + 3, + 3, + 4, + 2, + 7, + 4, + 4, + 2, + 3, + 4, + 2, + 2, + 4, + 1, + 1, + 4, + 2, + 2, + 4, + 2, + 3, + 3, + 2, + 6, + 2, + 5, + 2, + 4, + 2, + 2, + 1, + 3, + 6, + 5, + 3, + 2, + 4, + 1, + 2, + 1, + 4, + 1, + 4, + 3, + 1, + 4, + 5, + 1, + 2, + 2, + 1, + 2, + 4, + 3, + 1, + 2, + 4, + 1, + 1, + 3, + 1, + 4, + 1, + 3, + 3, + 2, + 3, + 2, + 2, + 3, + 2, + 3, + 3, + 1, + 3, + 2, + 3, + 3, + 2, + 2, + 8, + 2, + 4, + 4, + 4, + 5, + 7, + 3, + 1, + 1, + 4, + 1, + 3, + 3, + 3, + 4, + 4, + 1, + 5, + 1, + 2, + 3, + 1, + 3, + 3, + 5, + 5, + 1, + 3, + 2, + 4, + 5, + 5, + 6, + 4, + 5, + 3, + 3, + 5, + 3, + 1, + 2, + 3, + 2, + 7, + 2, + 8, + 2, + 5, + 3, + 9, + 4, + 1, + 3, + 4, + 2, + 1, + 1, + 2, + 7, + 1, + 3, + 3, + 3, + 4, + 2, + 2, + 4, + 1, + 3, + 2, + 2, + 2, + 4, + 6, + 3, + 1, + 2, + 4, + 2, + 2, + 3, + 2, + 4, + 4, + 2, + 1, + 1, + 2, + 2, + 5, + 3, + 1, + 2, + 3, + 5, + 2, + 2, + 1, + 4, + 4, + 2, + 3, + 1, + 5, + 5, + 1, + 2, + 4, + 3, + 1, + 4, + 2, + 4, + 1, + 3, + 1, + 1, + 2, + 3, + 4, + 3, + 2, + 2, + 5, + 2, + 1, + 8, + 3, + 2, + 4, + 5, + 1, + 1, + 6, + 2, + 4, + 4, + 1, + 7, + 3, + 4, + 2, + 3, + 1, + 2, + 1, + 3, + 4, + 1, + 4, + 5, + 1, + 2, + 2, + 5, + 2, + 4, + 2, + 2, + 1, + 2, + 4, + 3, + 6, + 3, + 2, + 2, + 2, + 2, + 2, + 4, + 1, + 4, + 3, + 3, + 1, + 1, + 4, + 3, + 5, + 4, + 2, + 2, + 4, + 2, + 1, + 3, + 3, + 2, + 1, + 2, + 5, + 2, + 3, + 2, + 3, + 1, + 2, + 5, + 3, + 1, + 3, + 4, + 6, + 3, + 2, + 4, + 3, + 3, + 2, + 5, + 3, + 2, + 5, + 2, + 5, + 2, + 1, + 2, + 2, + 3, + 4, + 4, + 2, + 2, + 3, + 4, + 2, + 6, + 3, + 9, + 3, + 4, + 2, + 4, + 1, + 1, + 4, + 2, + 3, + 3, + 4, + 4, + 2, + 6, + 5, + 3, + 4, + 4, + 3, + 1, + 4, + 2, + 3, + 6, + 2, + 1, + 1, + 4, + 2, + 4, + 4, + 2, + 1, + 1, + 1, + 3, + 3, + 4, + 4, + 2, + 2, + 5, + 5, + 2, + 2, + 2, + 2, + 1, + 2, + 2, + 1, + 6, + 5, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 5, + 2, + 1, + 3, + 3, + 4, + 4, + 3, + 2, + 2, + 1, + 4, + 3, + 3, + 3, + 1, + 2, + 2, + 3, + 4, + 4, + 2, + 3, + 4, + 3, + 6, + 4, + 1, + 6, + 2, + 3, + 3, + 1, + 5, + 5, + 4, + 4, + 4, + 2, + 6, + 3, + 3, + 2, + 3, + 3, + 5, + 2, + 2, + 3, + 1, + 1, + 2, + 2, + 1, + 5, + 4, + 2, + 3, + 3, + 3, + 2, + 3, + 3, + 1, + 5, + 2, + 2, + 2, + 2, + 1, + 3, + 1, + 2, + 2, + 1, + 3, + 5, + 2, + 3, + 3, + 7, + 2, + 2, + 3, + 5, + 5, + 2, + 3, + 1, + 4, + 2, + 1, + 3, + 6, + 3, + 2, + 4, + 2, + 3, + 2, + 1, + 2, + 2, + 2, + 4, + 2, + 1, + 1, + 6, + 2, + 3, + 3, + 1, + 3, + 4, + 3, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 4, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 5, + 3, + 2, + 3, + 1, + 2, + 2, + 3, + 2, + 3, + 2, + 3, + 1, + 2, + 2, + 6, + 1, + 3, + 4, + 3, + 3, + 1, + 4, + 4, + 5, + 4, + 3, + 1, + 1, + 3, + 2, + 3, + 2, + 1, + 4, + 2, + 2, + 3, + 2, + 1, + 3, + 2, + 5, + 2, + 2, + 3, + 2, + 4, + 1, + 2, + 3, + 3, + 2, + 1, + 3, + 3, + 1, + 3, + 3, + 1, + 1, + 2, + 1, + 1, + 5, + 5, + 4, + 5, + 2, + 4, + 2, + 1, + 2, + 2, + 3, + 2, + 2, + 1, + 5, + 3, + 8, + 2, + 5, + 1, + 2, + 3, + 3, + 1, + 1, + 2, + 3, + 2, + 3, + 4, + 1, + 3, + 1, + 5, + 1, + 2, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 3, + 2, + 3, + 2, + 5, + 1, + 4, + 5, + 3, + 1, + 2, + 5, + 2, + 3, + 2, + 4, + 3, + 4, + 1, + 1, + 2, + 3, + 3, + 3, + 5, + 4, + 2, + 1, + 4, + 4, + 4, + 3, + 3, + 3, + 2, + 2, + 2, + 2, + 4, + 1, + 5, + 1, + 2, + 4, + 3, + 5, + 1, + 1, + 4, + 2, + 4, + 1, + 2, + 1, + 1, + 6, + 4, + 3, + 5, + 2, + 2, + 3, + 5, + 2, + 2, + 3, + 5, + 2, + 1, + 4, + 4, + 4, + 5, + 1, + 2, + 3, + 2, + 2, + 3, + 2, + 2, + 6, + 2, + 2, + 3, + 3, + 3, + 2, + 2, + 2, + 2, + 4, + 2, + 3, + 2, + 3, + 4, + 3, + 3, + 1, + 4, + 3, + 3, + 1, + 2, + 3, + 1, + 3, + 3, + 3, + 2, + 1, + 3, + 3, + 1, + 6, + 5, + 1, + 1, + 4, + 2, + 4, + 3, + 5, + 5, + 6, + 6, + 4, + 3, + 3, + 2, + 3, + 3, + 2, + 2, + 3, + 1, + 5, + 2, + 3, + 5, + 4, + 1, + 4, + 2, + 1, + 3, + 2, + 1, + 4, + 3, + 5, + 3, + 2, + 2, + 2, + 2, + 4, + 6, + 3, + 5, + 3, + 1, + 1, + 2, + 1, + 6, + 4, + 2, + 3, + 4, + 2, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 4, + 3, + 3, + 4, + 3, + 3, + 4, + 4, + 1, + 4, + 4, + 1, + 2, + 2, + 5, + 4, + 2, + 2, + 3, + 1, + 2, + 6, + 2, + 5, + 3, + 1, + 1, + 1, + 1, + 2, + 4, + 2, + 3, + 1, + 2, + 1, + 5, + 3, + 1, + 1, + 2, + 3, + 2, + 3, + 2, + 6, + 3, + 1, + 2, + 6, + 4, + 3, + 3, + 6, + 2, + 3, + 3, + 1, + 2, + 2, + 4, + 3, + 3, + 5, + 1, + 2, + 1, + 3, + 5, + 1, + 4, + 4, + 7, + 3, + 2, + 2, + 2, + 2, + 3, + 1, + 1, + 4, + 2, + 2, + 1, + 3, + 2, + 3, + 5, + 1, + 2, + 2, + 1, + 4, + 1, + 3, + 3, + 1, + 1, + 3, + 2, + 5, + 1, + 2, + 5, + 3, + 6, + 2, + 3, + 2, + 4, + 5, + 3, + 3, + 5, + 2, + 4, + 3, + 2, + 3, + 6, + 1, + 3, + 3, + 1, + 1, + 3, + 5, + 4, + 2, + 5, + 3, + 2, + 5, + 3, + 1, + 1, + 1, + 2, + 4, + 4, + 1, + 1, + 2, + 5, + 3, + 4, + 1, + 4, + 5, + 3, + 1, + 6, + 1, + 3, + 6, + 2, + 1, + 4, + 3, + 5, + 3, + 1, + 5, + 3, + 1, + 3, + 5, + 2, + 2, + 3, + 5, + 4, + 1, + 5, + 6, + 2, + 5, + 1, + 7, + 1, + 1, + 3, + 3, + 1, + 2, + 2, + 1, + 3, + 4, + 1, + 5, + 6, + 3, + 3, + 3, + 2, + 1, + 1, + 3, + 2, + 4, + 2, + 2, + 3, + 2, + 3, + 4, + 4, + 1, + 2, + 5, + 2, + 1, + 3, + 1, + 1, + 3, + 3, + 4, + 4, + 1, + 2, + 3, + 3, + 1, + 4, + 1, + 3, + 1, + 3, + 1, + 2, + 3, + 1, + 2, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 4, + 3, + 3, + 3, + 1, + 2, + 1, + 2, + 2, + 4, + 2, + 3, + 2, + 2, + 3, + 3, + 2, + 7, + 2, + 3, + 1, + 4, + 1, + 4, + 4, + 1, + 1, + 2, + 4, + 4, + 2, + 1, + 4, + 1, + 3, + 1, + 3, + 2, + 1, + 3, + 3, + 3, + 2, + 1, + 3, + 2, + 3, + 4, + 2, + 1, + 2, + 3, + 4, + 5, + 4, + 2, + 2, + 2, + 1, + 4, + 3, + 1, + 2, + 4, + 3, + 2, + 1, + 5, + 4, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 6, + 2, + 4, + 3, + 4, + 4, + 2, + 1, + 2, + 3, + 2, + 2, + 2, + 2, + 3, + 1, + 1, + 4, + 5, + 1, + 1, + 3, + 2, + 1, + 1, + 3, + 1, + 4, + 3, + 1, + 3, + 3, + 5, + 3, + 4, + 2, + 1, + 2, + 2, + 1, + 2, + 4, + 3, + 3, + 3, + 1, + 3, + 2, + 5, + 4, + 3, + 1, + 3, + 3, + 2, + 3, + 2, + 1, + 3, + 5, + 6, + 4, + 3, + 2, + 2, + 2, + 2, + 4, + 2, + 2, + 2, + 6, + 1, + 5, + 2, + 4, + 3, + 2, + 4, + 3, + 1, + 1, + 2, + 2, + 2, + 3, + 4, + 4, + 2, + 2, + 3, + 3, + 3, + 3, + 1, + 2, + 2, + 4, + 1, + 4, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 4, + 2, + 2, + 2, + 5, + 3, + 1, + 3, + 1, + 6, + 2, + 2, + 1, + 3, + 3, + 3, + 3, + 2, + 3, + 2, + 4, + 4, + 1, + 3, + 3, + 1, + 2, + 3, + 1, + 2, + 7, + 6, + 3, + 1, + 2, + 2, + 5, + 3, + 1, + 5, + 4, + 3, + 4, + 4, + 1, + 3, + 1, + 2, + 3, + 4, + 6, + 4, + 2, + 4, + 4, + 3, + 1, + 1, + 1, + 2, + 3, + 2, + 3, + 2, + 1, + 3, + 1, + 6, + 3, + 3, + 5, + 3, + 3, + 3, + 1, + 1, + 3, + 3, + 5, + 1, + 1, + 4, + 3, + 3, + 4, + 2, + 3, + 2, + 2, + 5, + 1, + 2, + 3, + 1, + 3, + 3, + 1, + 2, + 1, + 1, + 2, + 2, + 4, + 2, + 4, + 6, + 3, + 2, + 1, + 2, + 2, + 4, + 2, + 2, + 1, + 2, + 6, + 2, + 1, + 2, + 2, + 5, + 5, + 4, + 4, + 1, + 3, + 3, + 1, + 5, + 3, + 4, + 1, + 2, + 4, + 3, + 3, + 1, + 1, + 3, + 3, + 1, + 1, + 3, + 2, + 2, + 4, + 1, + 1, + 2, + 3, + 3, + 2, + 2, + 1, + 3, + 1, + 1, + 2, + 3, + 3, + 2, + 4, + 3, + 1, + 2, + 3, + 4, + 4, + 4, + 1, + 3, + 2, + 6, + 1, + 3, + 3, + 3, + 5, + 4, + 4, + 2, + 3, + 2, + 5, + 2, + 1, + 6, + 2, + 2, + 2, + 2, + 2, + 1, + 5, + 1, + 3, + 2, + 2, + 3, + 5, + 1, + 3, + 2, + 6, + 2, + 2, + 5, + 6, + 3, + 2, + 3, + 3, + 5, + 2, + 2, + 2, + 1, + 4, + 2, + 4, + 3, + 4, + 2, + 1, + 3, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 3, + 2, + 1, + 2, + 2, + 3, + 1, + 1, + 2, + 4, + 1, + 5, + 1, + 2, + 3, + 5, + 4, + 4, + 3, + 3, + 5, + 2, + 4, + 3, + 1, + 5, + 2, + 2, + 3, + 2, + 2, + 1, + 2, + 4, + 3, + 1, + 2, + 1, + 3, + 2, + 4, + 1, + 4, + 3, + 2, + 2, + 3, + 3, + 3, + 2, + 1, + 2, + 2, + 1, + 2, + 4, + 5, + 3, + 4, + 3, + 3, + 3, + 3, + 2, + 3, + 1, + 3, + 6, + 5, + 1, + 3, + 3, + 3, + 1, + 2, + 2, + 2, + 2, + 4, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 4, + 3, + 3, + 4, + 2, + 4, + 4, + 1, + 1, + 4, + 1, + 3, + 3, + 3, + 2, + 4, + 2, + 5, + 2, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 3, + 3, + 1, + 1, + 5, + 2, + 1, + 2, + 2, + 2, + 5, + 2, + 3, + 3, + 3, + 4, + 3, + 1, + 2, + 1, + 3, + 4, + 3, + 2, + 2, + 2, + 1, + 2, + 5, + 4, + 4, + 2, + 1, + 1, + 3, + 3, + 2, + 5, + 1, + 2, + 1, + 2, + 3, + 1, + 5, + 2, + 1, + 3, + 1, + 1, + 3, + 5, + 3, + 4, + 3, + 1, + 2, + 2, + 7, + 2, + 2, + 3, + 2, + 2, + 1, + 1, + 1, + 3, + 3, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 3, + 1, + 2, + 3, + 2, + 3, + 1, + 2, + 4, + 1, + 2, + 4, + 3, + 3, + 1, + 6, + 1, + 1, + 2, + 5, + 3, + 3, + 6, + 2, + 3, + 1, + 1, + 1, + 3, + 5, + 3, + 3, + 2, + 2, + 1, + 2, + 1, + 4, + 2, + 1, + 2, + 5, + 1, + 2, + 2, + 3, + 3, + 1, + 3, + 3, + 3, + 1, + 6, + 2, + 2, + 3, + 2, + 4, + 2, + 4, + 1, + 2, + 2, + 2, + 3, + 3, + 2, + 3, + 3, + 1, + 2, + 1, + 3, + 2, + 2, + 1, + 3, + 2, + 3, + 1, + 2, + 2, + 1, + 1, + 1, + 3, + 2, + 2, + 2, + 3, + 2, + 3, + 3, + 2, + 3, + 3, + 1, + 3, + 1, + 2, + 2, + 2, + 1, + 3, + 1, + 1, + 3, + 5, + 7, + 3, + 1, + 3, + 2, + 5, + 3, + 7, + 3, + 2, + 5, + 3, + 2, + 4, + 1, + 1, + 4, + 2, + 2, + 3, + 3, + 2, + 4, + 1, + 3, + 1, + 3, + 1, + 1, + 5, + 1, + 2, + 1, + 1, + 8, + 2, + 2, + 4, + 2, + 1, + 3, + 4, + 2, + 2, + 2, + 1, + 3, + 3, + 2, + 1, + 1, + 2, + 3, + 1, + 1, + 3, + 3, + 3, + 6, + 1, + 5, + 4, + 4, + 1, + 2, + 1, + 3, + 2, + 1, + 1, + 3, + 1, + 6, + 1, + 1, + 1, + 3, + 1, + 2, + 2, + 2, + 5, + 2, + 3, + 4, + 4, + 3, + 1, + 1, + 3, + 3, + 5, + 5, + 1, + 1, + 4, + 3, + 3, + 2, + 1, + 3, + 4, + 5, + 1, + 1, + 2, + 1, + 4, + 1, + 1, + 3, + 4, + 1, + 4, + 2, + 4, + 3, + 3, + 2, + 2, + 2, + 5, + 2, + 2, + 2, + 2, + 4, + 2, + 3, + 5, + 1, + 1, + 5, + 3, + 1, + 2, + 3, + 3, + 2, + 5, + 2, + 2, + 1, + 5, + 4, + 1, + 3, + 1, + 1, + 6, + 2, + 5, + 2, + 4, + 2, + 3, + 3, + 3, + 2, + 4, + 2, + 2, + 4, + 3, + 2, + 3, + 1, + 3, + 1, + 7, + 6, + 1, + 2, + 7, + 3, + 2, + 1, + 1, + 3, + 2, + 1, + 2, + 2, + 1, + 1, + 2, + 4, + 2, + 4, + 2, + 3, + 2, + 1, + 1, + 2, + 4, + 4, + 4, + 3, + 1, + 1, + 1, + 5, + 3, + 5, + 1, + 2, + 1, + 2, + 2, + 1, + 3, + 2, + 2, + 2, + 2, + 1, + 2, + 3, + 3, + 2, + 2, + 1, + 1, + 2, + 3, + 1, + 3, + 2, + 1, + 3, + 2, + 2, + 5, + 1, + 4, + 2, + 8, + 1, + 2, + 2, + 2, + 3, + 2, + 1, + 6, + 2, + 3, + 2, + 1, + 4, + 1, + 1, + 3, + 2, + 1, + 2, + 2, + 2, + 1, + 2, + 6, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 2, + 5, + 4, + 4, + 1, + 2, + 3, + 1, + 2, + 2, + 3, + 1, + 1, + 2, + 3, + 1, + 3, + 2, + 3, + 3, + 1, + 1, + 2, + 2, + 2, + 2, + 5, + 2, + 4, + 2, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 3, + 3, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 4, + 2, + 2, + 5, + 1, + 2, + 1, + 3, + 2, + 1, + 1, + 2, + 3, + 3, + 3, + 1, + 2, + 2, + 1, + 1, + 4, + 2, + 2, + 2, + 4, + 1, + 1, + 2, + 1, + 4, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 4, + 4, + 6, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 4, + 1, + 4, + 4, + 2, + 5, + 2, + 2, + 2, + 1, + 3, + 1, + 1, + 2, + 2, + 2, + 2, + 1, + 3, + 1, + 3, + 6, + 4, + 3, + 1, + 1, + 3, + 3, + 3, + 5, + 1, + 5, + 3, + 1, + 1, + 4, + 2, + 1, + 1, + 3, + 3, + 2, + 4, + 4, + 3, + 1, + 1, + 3, + 2, + 2, + 1, + 3, + 2, + 6, + 2, + 2, + 4, + 2, + 1, + 2, + 1, + 1, + 3, + 4, + 4, + 1, + 1, + 1, + 2, + 2, + 5, + 1, + 2, + 3, + 2, + 4, + 3, + 6, + 3, + 3, + 3, + 4, + 2, + 3, + 1, + 1, + 3, + 1, + 3, + 2, + 3, + 3, + 3, + 1, + 2, + 2, + 3, + 2, + 5, + 3, + 1, + 2, + 2, + 2, + 1, + 3, + 3, + 1, + 4, + 3, + 1, + 3, + 1, + 3, + 3, + 1, + 3, + 5, + 4, + 3, + 2, + 3, + 2, + 1, + 2, + 1, + 3, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 4, + 5, + 2, + 1, + 1, + 3, + 1, + 2, + 3, + 2, + 2, + 4, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 5, + 1, + 2, + 3, + 1, + 2, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 5, + 3, + 2, + 1, + 1, + 1, + 3, + 1, + 3, + 3, + 3, + 2, + 1, + 3, + 4, + 5, + 2, + 2, + 2, + 1, + 3, + 3, + 1, + 2, + 3, + 2, + 2, + 2, + 4, + 1, + 1, + 1, + 3, + 1, + 3, + 1, + 2, + 3, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 3, + 1, + 2, + 4, + 2, + 3, + 1, + 4, + 2, + 5, + 2, + 3, + 1, + 1, + 1, + 3, + 2, + 3, + 3, + 5, + 5, + 5, + 2, + 3, + 1, + 1, + 1, + 1, + 2, + 4, + 3, + 3, + 5, + 1, + 3, + 1, + 3, + 2, + 4, + 3, + 2, + 4, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 4, + 3, + 1, + 4, + 1, + 1, + 1, + 4, + 3, + 3, + 2, + 2, + 2, + 4, + 2, + 4, + 2, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 2, + 4, + 1, + 2, + 2, + 2, + 5, + 3, + 1, + 1, + 3, + 2, + 1, + 6, + 2, + 3, + 4, + 4, + 1, + 1, + 1, + 3, + 2, + 2, + 1, + 2, + 3, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 5, + 3, + 4, + 2, + 2, + 1, + 3, + 1, + 2, + 1, + 2, + 2, + 6, + 3, + 1, + 3, + 2, + 2, + 2, + 1, + 3, + 1, + 3, + 3, + 2, + 3, + 3, + 2, + 1, + 1, + 4, + 2, + 1, + 1, + 3, + 1, + 2, + 3, + 2, + 1, + 1, + 2, + 3, + 1, + 2, + 1, + 3, + 1, + 3, + 2, + 2, + 3, + 2, + 3, + 2, + 4, + 4, + 1, + 3, + 1, + 3, + 3, + 2, + 2, + 1, + 3, + 3, + 2, + 2, + 3, + 1, + 1, + 2, + 2, + 2, + 1, + 3, + 2, + 1, + 2, + 1, + 2, + 4, + 1, + 2, + 1, + 1, + 4, + 1, + 3, + 3, + 3, + 1, + 1, + 2, + 2, + 1, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 3, + 2, + 2, + 3, + 4, + 1, + 1, + 1, + 1, + 4, + 4, + 3, + 3, + 1, + 1, + 2, + 5, + 2, + 1, + 1, + 4, + 2, + 2, + 1, + 3, + 3, + 2, + 3, + 5, + 1, + 1, + 3, + 2, + 2, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 2, + 2, + 2, + 3, + 2, + 4, + 1, + 2, + 3, + 1, + 1, + 4, + 2, + 2, + 2, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 3, + 3, + 1, + 3, + 4, + 4, + 3, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 3, + 4, + 2, + 3, + 1, + 5, + 2, + 6, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 3, + 3, + 4, + 3, + 1, + 2, + 1, + 3, + 2, + 2, + 2, + 5, + 1, + 1, + 3, + 1, + 5, + 2, + 2, + 2, + 5, + 4, + 2, + 3, + 2, + 2, + 1, + 1, + 6, + 2, + 2, + 1, + 2, + 2, + 2, + 3, + 1, + 1, + 5, + 4, + 1, + 2, + 2, + 3, + 3, + 5, + 4, + 5, + 2, + 1, + 2, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 3, + 1, + 1, + 5, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 3, + 5, + 2, + 3, + 2, + 2, + 1, + 2, + 1, + 2, + 2, + 2, + 2, + 1, + 2, + 4, + 2, + 1, + 1, + 2, + 1, + 3, + 2, + 2, + 2, + 3, + 3, + 1, + 2, + 4, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 4, + 1, + 2, + 4, + 2, + 2, + 2, + 1, + 4, + 2, + 2, + 2, + 1, + 3, + 5, + 1, + 1, + 4, + 1, + 3, + 1, + 3, + 1, + 4, + 3, + 1, + 1, + 2, + 1, + 5, + 3, + 1, + 1, + 4, + 4, + 2, + 1, + 4, + 4, + 6, + 4, + 1, + 3, + 1, + 1, + 4, + 4, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 4, + 1, + 2, + 1, + 2, + 4, + 2, + 2, + 2, + 3, + 2, + 1, + 4, + 4, + 3, + 3, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 5, + 1, + 2, + 1, + 2, + 3, + 1, + 2, + 4, + 1, + 2, + 2, + 3, + 2, + 2, + 2, + 1, + 1, + 3, + 1, + 3, + 2, + 1, + 2, + 2, + 3, + 3, + 3, + 2, + 1, + 2, + 2, + 1, + 2, + 5, + 2, + 4, + 3, + 1, + 4, + 2, + 3, + 4, + 1, + 1, + 4, + 4, + 4, + 3, + 2, + 2, + 2, + 1, + 1, + 1, + 2, + 3, + 1, + 2, + 2, + 5, + 1, + 2, + 2, + 3, + 2, + 3, + 5, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 3, + 3, + 3, + 1, + 1, + 3, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 4, + 4, + 2, + 1, + 2, + 3, + 1, + 1, + 1, + 4, + 3, + 1, + 4, + 2, + 1, + 2, + 2, + 4, + 2, + 3, + 3, + 1, + 3, + 1, + 1, + 1, + 2, + 4, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 3, + 4, + 2, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 5, + 1, + 1, + 3, + 2, + 2, + 3, + 1, + 2, + 3, + 3, + 2, + 3, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 2, + 2, + 4, + 3, + 2, + 1, + 1, + 1, + 1, + 2, + 4, + 2, + 1, + 4, + 1, + 2, + 4, + 1, + 3, + 2, + 3, + 3, + 4, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 2, + 5, + 6, + 5, + 2, + 3, + 1, + 2, + 4, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 4, + 2, + 1, + 3, + 1, + 1, + 3, + 4, + 3, + 1, + 2, + 2, + 1, + 2, + 2, + 3, + 2, + 3, + 2, + 2, + 2, + 3, + 1, + 3, + 1, + 4, + 2, + 1, + 1, + 1, + 4, + 1, + 6, + 2, + 1, + 4, + 1, + 3, + 2, + 2, + 3, + 1, + 2, + 1, + 2, + 1, + 4, + 7, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 3, + 2, + 3, + 2, + 1, + 4, + 1, + 4, + 2, + 3, + 2, + 3, + 2, + 3, + 2, + 1, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 4, + 1, + 1, + 1, + 4, + 3, + 3, + 2, + 2, + 1, + 1, + 1, + 3, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 2, + 2, + 1, + 4, + 1, + 3, + 3, + 1, + 1, + 2, + 2, + 1, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 5, + 2, + 3, + 4, + 1, + 2, + 2, + 4, + 1, + 4, + 2, + 3, + 1, + 3, + 1, + 1, + 3, + 2, + 4, + 3, + 2, + 5, + 2, + 4, + 4, + 3, + 2, + 1, + 2, + 3, + 1, + 3, + 1, + 2, + 2, + 3, + 2, + 1, + 4, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 3, + 2, + 1, + 2, + 2, + 2, + 5, + 2, + 1, + 1, + 1, + 1, + 4, + 2, + 2, + 3, + 1, + 1, + 1, + 1, + 4, + 5, + 3, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 3, + 2, + 2, + 3, + 3, + 1, + 1, + 2, + 1, + 1, + 2, + 3, + 2, + 1, + 4, + 1, + 2, + 2, + 3, + 1, + 2, + 2, + 2, + 2, + 2, + 2, + 3, + 3, + 3, + 2, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 3, + 2, + 5, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 4, + 3, + 2, + 2, + 3, + 3, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 3, + 2, + 1, + 2, + 4, + 2, + 1, + 2, + 1, + 1, + 3, + 1, + 2, + 1, + 2, + 1, + 3, + 1, + 3, + 2, + 1, + 1, + 3, + 2, + 3, + 2, + 2, + 2, + 1, + 4, + 1, + 4, + 3, + 2, + 2, + 1, + 1, + 3, + 1, + 4, + 1, + 1, + 2, + 1, + 2, + 2, + 2, + 1, + 4, + 6, + 1, + 3, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 4, + 1, + 2, + 4, + 2, + 2, + 3, + 1, + 3, + 2, + 4, + 2, + 5, + 6, + 3, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 2, + 1, + 8, + 4, + 2, + 4, + 2, + 2, + 2, + 1, + 2, + 4, + 4, + 2, + 3, + 2, + 5, + 2, + 4, + 1, + 1, + 2, + 4, + 4, + 3, + 1, + 2, + 3, + 2, + 2, + 1, + 3, + 3, + 2, + 1, + 3, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 3, + 2, + 2, + 1, + 1, + 1, + 4, + 2, + 2, + 1, + 4, + 3, + 2, + 1, + 1, + 3, + 2, + 2, + 3, + 3, + 5, + 2, + 2, + 1, + 6, + 1, + 6, + 1, + 1, + 2, + 2, + 2, + 1, + 6, + 2, + 1, + 2, + 2, + 4, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 4, + 2, + 2, + 1, + 4, + 2, + 2, + 1, + 3, + 1, + 1, + 2, + 4, + 2, + 1, + 3, + 4, + 3, + 2, + 2, + 1, + 2, + 3, + 1, + 4, + 7, + 2, + 3, + 1, + 2, + 1, + 1, + 4, + 2, + 1, + 1, + 2, + 1, + 3, + 4, + 2, + 1, + 5, + 2, + 1, + 2, + 2, + 1, + 1, + 2, + 3, + 4, + 4, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 2, + 2, + 1, + 3, + 1, + 3, + 1, + 3, + 3, + 1, + 2, + 2, + 2, + 4, + 1, + 1, + 1, + 2, + 3, + 1, + 2, + 3, + 1, + 3, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 5, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 6, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 4, + 2, + 1, + 2, + 2, + 3, + 1, + 1, + 5, + 1, + 2, + 1, + 1, + 2, + 3, + 1, + 1, + 3, + 1, + 4, + 4, + 4, + 1, + 2, + 2, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 2, + 3, + 3, + 1, + 3, + 4, + 2, + 1, + 2, + 1, + 2, + 2, + 3, + 2, + 1, + 4, + 1, + 1, + 1, + 5, + 2, + 2, + 1, + 1, + 3, + 3, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 4, + 1, + 2, + 2, + 4, + 2, + 3, + 4, + 1, + 1, + 5, + 2, + 2, + 1, + 3, + 1, + 3, + 3, + 3, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 2, + 2, + 1, + 4, + 1, + 2, + 1, + 5, + 3, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 2, + 1, + 2, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 6, + 5, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 3, + 1, + 2, + 1, + 5, + 1, + 1, + 1, + 1, + 2, + 4, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 3, + 1, + 3, + 2, + 1, + 3, + 1, + 3, + 2, + 1, + 2, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 4, + 2, + 1, + 1, + 1, + 2, + 1, + 4, + 2, + 2, + 1, + 1, + 4, + 1, + 2, + 1, + 4, + 1, + 6, + 3, + 4, + 5, + 2, + 2, + 1, + 3, + 3, + 2, + 2, + 2, + 1, + 2, + 4, + 2, + 2, + 1, + 2, + 3, + 1, + 1, + 1, + 2, + 1, + 1, + 5, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 2, + 1, + 3, + 2, + 3, + 4, + 4, + 1, + 4, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 3, + 4, + 1, + 2, + 2, + 4, + 3, + 2, + 2, + 2, + 5, + 1, + 1, + 2, + 3, + 2, + 1, + 1, + 1, + 1, + 7, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 2, + 3, + 2, + 2, + 2, + 3, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 2, + 1, + 3, + 3, + 3, + 1, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 3, + 1, + 1, + 2, + 1, + 2, + 4, + 1, + 2, + 2, + 3, + 1, + 2, + 5, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 3, + 2, + 1, + 3, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 4, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 4, + 2, + 1, + 1, + 1, + 3, + 1, + 3, + 1, + 1, + 2, + 2, + 2, + 4, + 1, + 3, + 4, + 3, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 4, + 2, + 1, + 1, + 4, + 3, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 3, + 2, + 3, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 3, + 1, + 3, + 2, + 1, + 1, + 1, + 3, + 1, + 3, + 1, + 3, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 2, + 6, + 2, + 1, + 2, + 2, + 2, + 1, + 3, + 2, + 2, + 2, + 1, + 4, + 1, + 3, + 3, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 4, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 1, + 3, + 2, + 3, + 3, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 3, + 2, + 1, + 2, + 1, + 2, + 3, + 2, + 1, + 1, + 1, + 2, + 5, + 1, + 3, + 3, + 2, + 2, + 1, + 1, + 2, + 2, + 4, + 2, + 2, + 3, + 2, + 2, + 2, + 1, + 4, + 1, + 3, + 2, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 4, + 1, + 1, + 1, + 3, + 2, + 2, + 1, + 3, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 3, + 4, + 1, + 1, + 1, + 2, + 1, + 4, + 1, + 5, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 2, + 2, + 3, + 2, + 2, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 4, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 4, + 4, + 1, + 3, + 2, + 3, + 2, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 3, + 1, + 2, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 4, + 2, + 1, + 1, + 2, + 3, + 4, + 2, + 2, + 1, + 2, + 2, + 3, + 2, + 1, + 1, + 1, + 1, + 5, + 1, + 3, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 2, + 1, + 1, + 3, + 2, + 1, + 3, + 2, + 4, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 3, + 4, + 3, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 4, + 3, + 2, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 2, + 4, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 4, + 2, + 2, + 1, + 1, + 1, + 1, + 17, + 1, + 1, + 2, + 3, + 2, + 1, + 2, + 1, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 2, + 4, + 1, + 1, + 1, + 3, + 4, + 2, + 1, + 1, + 3, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 3, + 1, + 1, + 1, + 3, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 3, + 1, + 1, + 3, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 4, + 4, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 3, + 1, + 2, + 2, + 1, + 2, + 3, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 3, + 1, + 1, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 5, + 1, + 1, + 1, + 2, + 4, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 3, + 1, + 2, + 1, + 2, + 3, + 1, + 2, + 2, + 2, + 4, + 1, + 3, + 1, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 3, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 3, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 2, + 3, + 2, + 1, + 3, + 2, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 3, + 5, + 1, + 1, + 4, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 3, + 2, + 4, + 3, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 3, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 2, + 3, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 3, + 1, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 3, + 2, + 2, + 2, + 3, + 2, + 2, + 1, + 1, + 4, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 3, + 1, + 1, + 4, + 1, + 1, + 1, + 2, + 3, + 4, + 4, + 1, + 2, + 2, + 2, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 3, + 4, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 3, + 1, + 1, + 1, + 3, + 3, + 3, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 4, + 2, + 1, + 1, + 4, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 4, + 2, + 2, + 1, + 1, + 3, + 2, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 4, + 1, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 4, + 2, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 4, + 1, + 1, + 2, + 3, + 1, + 2, + 2, + 1, + 4, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 3, + 1, + 3, + 1, + 3, + 1, + 1, + 3, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 3, + 1, + 2, + 3, + 1, + 3, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 3, + 2, + 1, + 2, + 2, + 2, + 2, + 5, + 2, + 1, + 1, + 2, + 3, + 2, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 3, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 3, + 3, + 3, + 1, + 3, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 3, + 1, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 3, + 3, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 3, + 2, + 1, + 2, + 2, + 2, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 3, + 3, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 5, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 2, + 2, + 2, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 3, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 4, + 1, + 4, + 2, + 2, + 1, + 1, + 3, + 1, + 2, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 3, + 2, + 1, + 2, + 1, + 3, + 2, + 1, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 1, + 4, + 3, + 1, + 2, + 2, + 2, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 3, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 3, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 2, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 2, + 2, + 2, + 4, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 3, + 2, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 3, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 3, + 1, + 3, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 3, + 3, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 3, + 3, + 1, + 1, + 2, + 1, + 2, + 3, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 4, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 2, + 2, + 1, + 4, + 2, + 1, + 1, + 2, + 1, + 2, + 2, + 3, + 2, + 1, + 1, + 1, + 2, + 3, + 2, + 2, + 1, + 3, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 4, + 2, + 3, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 2, + 3, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 2, + 1, + 4, + 1, + 1, + 1, + 1, + 1, + 2, + 4, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 4, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 2, + 3, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 4, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 2, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 3, + 1, + 4, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 3, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 4, + 1, + 3, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 3, + 1, + 3, + 1, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 4, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 3, + 1, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 3, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 1, + 4, + 1, + 1, + 1, + 3, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 3, + 2, + 2, + 1, + 2, + 2, + 1, + 3, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 4, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 2, + 3, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 3, + 2, + 1, + 2, + 1, + 3, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 3, + 2, + 1, + 1, + 3, + 1, + 3, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 4, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 3, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 2, + 3, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 3, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 3, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 3, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 2, + 3, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 4, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 3, + 1, + 2, + 1, + 2, + 2, + 2, + 1, + 3, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 4, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 4, + 1, + 1, + 2, + 3, + 1, + 3, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 2, + 1, + 4, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 4, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 4, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 3, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 3, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 4, + 2, + 3, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 5, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 3, + 1, + 3, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 2, + 4, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 2, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 2, + 2, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 3, + 2, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 3, + 1, + 4, + 1, + 1, + 1, + 2, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 4, + 3, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 2, + 3, + 1, + 2, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 2, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 4, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 2, + 2, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 2, + 2, + 1, + 3, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 4, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 2, + 2, + 2, + 3, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 4, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 4, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 3, + 2, + 2, + 1, + 1, + 3, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 2, + 3, + 1, + 1, + 2, + 1, + 1, + 3, + 3, + 3, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 2, + 3, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 2, + 2, + 3, + 1, + 1, + 2, + 3, + 2, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 3, + 2, + 3, + 1, + 2, + 2, + 3, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 3, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 3, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 4, + 1, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 3, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 3, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 4, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 2, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 3, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 4, + 1, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 3, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 4, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 3, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 3, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 15, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "valSizeRange": [ + 127, + 159, + 191, + 287, + 319, + 351, + 415, + 447, + 479, + 511, + 543, + 575, + 607, + 639, + 671, + 703, + 735, + 767, + 799, + 831, + 863, + 895, + 927, + 959, + 991, + 1023, + 1055, + 1087, + 1119, + 1151, + 1183, + 1215, + 1247, + 1279, + 1311, + 1343, + 1375, + 1407, + 1439, + 1471, + 1503, + 1535, + 1567, + 1599, + 1631, + 1663, + 1695, + 1727, + 1759, + 1791, + 1823, + 1855, + 1887, + 1919, + 1951, + 1983, + 2015, + 2047, + 2079, + 2111, + 2143, + 2175, + 2207, + 2239, + 2271, + 2303, + 2335, + 2367, + 2399, + 2431, + 2463, + 2495, + 2527, + 2559, + 2591, + 2623, + 2655, + 2687, + 2719, + 2751, + 2783, + 2815, + 2847, + 2879, + 2911, + 2943, + 2975, + 3007, + 3039, + 3071, + 3103, + 3135, + 3167, + 3199, + 3231, + 3263, + 3295, + 3327, + 3359, + 3391, + 3423, + 3455, + 3487, + 3519, + 3551, + 3583, + 3615, + 3647, + 3679, + 3711, + 3743, + 3775, + 3807, + 3839, + 3871, + 3903, + 3935, + 3967, + 3999, + 4031, + 4063, + 4095, + 4127, + 4159, + 4191, + 4223, + 4255, + 4287, + 4319, + 4351, + 4383, + 4415, + 4447, + 4479, + 4511, + 4543, + 4575, + 4607, + 4639, + 4671, + 4703, + 4735, + 4767, + 4799, + 4831, + 4863, + 4895, + 4927, + 4959, + 4991, + 5023, + 5055, + 5087, + 5119, + 5151, + 5183, + 5215, + 5247, + 5279, + 5311, + 5343, + 5375, + 5407, + 5439, + 5471, + 5503, + 5535, + 5567, + 5599, + 5631, + 5663, + 5695, + 5727, + 5759, + 5791, + 5823, + 5855, + 5887, + 5919, + 5951, + 5983, + 6015, + 6047, + 6079, + 6111, + 6143, + 6175, + 6207, + 6239, + 6271, + 6303, + 6335, + 6367, + 6399, + 6431, + 6463, + 6495, + 6527, + 6559, + 6591, + 6623, + 6655, + 6687, + 6719, + 6751, + 6783, + 6815, + 6847, + 6879, + 6911, + 6943, + 6975, + 7007, + 7039, + 7071, + 7103, + 7135, + 7167, + 7199, + 7231, + 7263, + 7295, + 7327, + 7359, + 7391, + 7423, + 7455, + 7487, + 7519, + 7551, + 7583, + 7615, + 7647, + 7679, + 7711, + 7743, + 7775, + 7807, + 7839, + 7871, + 7903, + 7935, + 7967, + 7999, + 8031, + 8063, + 8095, + 8127, + 8159, + 8191, + 8223, + 8255, + 8287, + 8319, + 8351, + 8383, + 8415, + 8447, + 8479, + 8511, + 8543, + 8575, + 8607, + 8639, + 8671, + 8703, + 8735, + 8767, + 8799, + 8831, + 8863, + 8895, + 8927, + 8959, + 8991, + 9023, + 9055, + 9087, + 9119, + 9151, + 9183, + 9215, + 9247, + 9279, + 9311, + 9343, + 9375, + 9407, + 9439, + 9471, + 9503, + 9535, + 9567, + 9599, + 9631, + 9663, + 9695, + 9727, + 9759, + 9791, + 9823, + 9855, + 9887, + 9919, + 9951, + 9983, + 10015, + 10047, + 10079, + 10111, + 10143, + 10175, + 10207, + 10239, + 10271, + 10303, + 10335, + 10367, + 10399, + 10431, + 10463, + 10495, + 10527, + 10559, + 10591, + 10623, + 10655, + 10687, + 10719, + 10751, + 10783, + 10815, + 10847, + 10879, + 10911, + 10943, + 10975, + 11007, + 11039, + 11071, + 11103, + 11135, + 11167, + 11199, + 11231, + 11263, + 11295, + 11327, + 11359, + 11391, + 11423, + 11455, + 11487, + 11519, + 11551, + 11583, + 11615, + 11647, + 11679, + 11711, + 11743, + 11775, + 11807, + 11839, + 11871, + 11903, + 11935, + 11967, + 11999, + 12031, + 12063, + 12095, + 12127, + 12159, + 12191, + 12223, + 12255, + 12287, + 12319, + 12351, + 12383, + 12415, + 12447, + 12479, + 12511, + 12543, + 12575, + 12607, + 12639, + 12671, + 12703, + 12735, + 12767, + 12799, + 12831, + 12863, + 12895, + 12927, + 12959, + 12991, + 13023, + 13055, + 13087, + 13119, + 13151, + 13183, + 13215, + 13247, + 13279, + 13311, + 13343, + 13375, + 13407, + 13439, + 13471, + 13503, + 13535, + 13567, + 13599, + 13631, + 13663, + 13695, + 13727, + 13759, + 13791, + 13823, + 13855, + 13887, + 13919, + 13951, + 13983, + 14015, + 14047, + 14079, + 14111, + 14143, + 14175, + 14207, + 14239, + 14271, + 14303, + 14335, + 14367, + 14399, + 14431, + 14463, + 14495, + 14527, + 14559, + 14591, + 14623, + 14655, + 14687, + 14719, + 14751, + 14783, + 14815, + 14847, + 14879, + 14911, + 14943, + 14975, + 15007, + 15039, + 15071, + 15103, + 15135, + 15167, + 15199, + 15231, + 15263, + 15295, + 15327, + 15359, + 15391, + 15423, + 15455, + 15487, + 15519, + 15551, + 15583, + 15615, + 15647, + 15679, + 15711, + 15743, + 15775, + 15807, + 15839, + 15871, + 15903, + 15935, + 15967, + 15999, + 16031, + 16063, + 16095, + 16127, + 16159, + 16191, + 16223, + 16255, + 16287, + 16319, + 16351, + 16383, + 16415, + 16447, + 16479, + 16511, + 16543, + 16575, + 16607, + 16639, + 16671, + 16703, + 16735, + 16767, + 16799, + 16831, + 16863, + 16895, + 16927, + 16959, + 16991, + 17023, + 17055, + 17087, + 17119, + 17151, + 17183, + 17215, + 17247, + 17279, + 17311, + 17343, + 17375, + 17407, + 17439, + 17471, + 17503, + 17535, + 17567, + 17599, + 17631, + 17663, + 17695, + 17727, + 17759, + 17791, + 17823, + 17855, + 17887, + 17919, + 17951, + 17983, + 18015, + 18047, + 18079, + 18111, + 18143, + 18175, + 18207, + 18239, + 18271, + 18303, + 18335, + 18367, + 18399, + 18431, + 18463, + 18495, + 18527, + 18559, + 18591, + 18623, + 18655, + 18687, + 18719, + 18751, + 18783, + 18815, + 18847, + 18879, + 18911, + 18943, + 18975, + 19007, + 19039, + 19071, + 19103, + 19135, + 19167, + 19199, + 19231, + 19263, + 19295, + 19327, + 19359, + 19391, + 19423, + 19455, + 19487, + 19519, + 19551, + 19583, + 19615, + 19647, + 19679, + 19711, + 19743, + 19775, + 19807, + 19839, + 19871, + 19903, + 19935, + 19967, + 19999, + 20031, + 20063, + 20095, + 20127, + 20159, + 20191, + 20223, + 20255, + 20287, + 20319, + 20351, + 20383, + 20415, + 20447, + 20479, + 20511, + 20543, + 20575, + 20607, + 20639, + 20671, + 20703, + 20735, + 20767, + 20799, + 20831, + 20863, + 20895, + 20927, + 20959, + 20991, + 21023, + 21055, + 21087, + 21119, + 21151, + 21183, + 21215, + 21247, + 21279, + 21311, + 21343, + 21375, + 21407, + 21439, + 21471, + 21503, + 21535, + 21567, + 21599, + 21631, + 21663, + 21695, + 21727, + 21759, + 21791, + 21823, + 21855, + 21887, + 21919, + 21951, + 21983, + 22015, + 22047, + 22079, + 22111, + 22143, + 22175, + 22207, + 22239, + 22271, + 22303, + 22335, + 22367, + 22399, + 22431, + 22463, + 22495, + 22527, + 22559, + 22591, + 22623, + 22655, + 22687, + 22719, + 22751, + 22783, + 22815, + 22847, + 22879, + 22911, + 22943, + 22975, + 23007, + 23039, + 23071, + 23103, + 23135, + 23167, + 23199, + 23231, + 23263, + 23295, + 23327, + 23359, + 23391, + 23423, + 23455, + 23487, + 23519, + 23551, + 23583, + 23615, + 23647, + 23679, + 23711, + 23743, + 23775, + 23807, + 23839, + 23871, + 23903, + 23935, + 23967, + 23999, + 24031, + 24063, + 24095, + 24127, + 24159, + 24191, + 24223, + 24255, + 24287, + 24319, + 24351, + 24383, + 24415, + 24447, + 24479, + 24511, + 24543, + 24575, + 24607, + 24639, + 24671, + 24703, + 24735, + 24767, + 24799, + 24831, + 24863, + 24895, + 24927, + 24959, + 24991, + 25023, + 25055, + 25087, + 25119, + 25151, + 25183, + 25215, + 25247, + 25279, + 25311, + 25343, + 25375, + 25407, + 25439, + 25471, + 25503, + 25535, + 25567, + 25599, + 25631, + 25663, + 25695, + 25727, + 25759, + 25791, + 25823, + 25855, + 25887, + 25919, + 25951, + 25983, + 26015, + 26047, + 26079, + 26111, + 26143, + 26175, + 26207, + 26239, + 26271, + 26303, + 26335, + 26367, + 26399, + 26431, + 26463, + 26495, + 26527, + 26559, + 26591, + 26623, + 26655, + 26687, + 26719, + 26751, + 26783, + 26815, + 26847, + 26879, + 26911, + 26943, + 26975, + 27007, + 27039, + 27071, + 27103, + 27135, + 27167, + 27199, + 27231, + 27263, + 27295, + 27327, + 27359, + 27391, + 27423, + 27455, + 27487, + 27519, + 27551, + 27583, + 27615, + 27647, + 27679, + 27711, + 27743, + 27775, + 27807, + 27839, + 27871, + 27903, + 27935, + 27967, + 27999, + 28031, + 28063, + 28095, + 28127, + 28159, + 28191, + 28223, + 28255, + 28287, + 28319, + 28351, + 28383, + 28415, + 28447, + 28479, + 28511, + 28543, + 28575, + 28607, + 28639, + 28671, + 28703, + 28735, + 28767, + 28799, + 28831, + 28863, + 28895, + 28927, + 28959, + 28991, + 29023, + 29055, + 29087, + 29119, + 29151, + 29183, + 29215, + 29247, + 29279, + 29311, + 29343, + 29375, + 29407, + 29439, + 29471, + 29503, + 29535, + 29567, + 29599, + 29631, + 29663, + 29695, + 29727, + 29759, + 29791, + 29823, + 29855, + 29887, + 29919, + 29951, + 29983, + 30015, + 30047, + 30079, + 30111, + 30143, + 30175, + 30207, + 30239, + 30271, + 30303, + 30335, + 30367, + 30399, + 30431, + 30463, + 30495, + 30527, + 30559, + 30591, + 30623, + 30655, + 30687, + 30719, + 30751, + 30783, + 30815, + 30847, + 30879, + 30911, + 30943, + 30975, + 31007, + 31039, + 31071, + 31103, + 31135, + 31167, + 31199, + 31231, + 31263, + 31295, + 31327, + 31359, + 31391, + 31423, + 31455, + 31487, + 31519, + 31551, + 31583, + 31615, + 31647, + 31679, + 31711, + 31743, + 31775, + 31807, + 31839, + 31871, + 31903, + 31935, + 31967, + 31999, + 32031, + 32063, + 32095, + 32127, + 32159, + 32191, + 32223, + 32255, + 32287, + 32319, + 32351, + 32383, + 32415, + 32447, + 32479, + 32511, + 32543, + 32575, + 32607, + 32639, + 32671, + 32703, + 32735, + 32767, + 32799, + 32831, + 32863, + 32895, + 32927, + 32959, + 32991, + 33023, + 33055, + 33087, + 33119, + 33151, + 33183, + 33215, + 33247, + 33279, + 33311, + 33343, + 33375, + 33407, + 33439, + 33471, + 33503, + 33535, + 33567, + 33599, + 33631, + 33663, + 33695, + 33727, + 33759, + 33791, + 33823, + 33855, + 33887, + 33919, + 33951, + 33983, + 34015, + 34047, + 34079, + 34111, + 34143, + 34175, + 34207, + 34239, + 34271, + 34303, + 34335, + 34367, + 34399, + 34431, + 34463, + 34495, + 34527, + 34559, + 34591, + 34623, + 34655, + 34687, + 34719, + 34751, + 34783, + 34815, + 34847, + 34879, + 34911, + 34943, + 34975, + 35007, + 35039, + 35071, + 35103, + 35135, + 35167, + 35199, + 35231, + 35263, + 35295, + 35327, + 35359, + 35391, + 35423, + 35455, + 35487, + 35519, + 35551, + 35583, + 35615, + 35647, + 35679, + 35711, + 35743, + 35775, + 35807, + 35839, + 35871, + 35903, + 35935, + 35967, + 35999, + 36031, + 36063, + 36095, + 36127, + 36159, + 36191, + 36223, + 36255, + 36287, + 36319, + 36351, + 36383, + 36415, + 36447, + 36479, + 36511, + 36543, + 36575, + 36607, + 36639, + 36671, + 36703, + 36735, + 36767, + 36799, + 36831, + 36863, + 36895, + 36927, + 36959, + 36991, + 37023, + 37055, + 37087, + 37119, + 37151, + 37183, + 37215, + 37247, + 37279, + 37311, + 37343, + 37375, + 37407, + 37439, + 37471, + 37503, + 37535, + 37567, + 37599, + 37631, + 37663, + 37695, + 37727, + 37759, + 37791, + 37823, + 37855, + 37887, + 37919, + 37951, + 37983, + 38015, + 38047, + 38079, + 38111, + 38143, + 38175, + 38207, + 38239, + 38271, + 38303, + 38335, + 38367, + 38399, + 38431, + 38463, + 38495, + 38527, + 38559, + 38591, + 38623, + 38655, + 38687, + 38719, + 38751, + 38783, + 38815, + 38847, + 38879, + 38911, + 38943, + 38975, + 39007, + 39039, + 39071, + 39103, + 39135, + 39167, + 39199, + 39231, + 39263, + 39295, + 39327, + 39359, + 39391, + 39423, + 39455, + 39487, + 39519, + 39551, + 39583, + 39615, + 39647, + 39679, + 39711, + 39743, + 39775, + 39807, + 39839, + 39871, + 39903, + 39935, + 39967, + 39999, + 40031, + 40063, + 40095, + 40127, + 40159, + 40191, + 40223, + 40255, + 40287, + 40319, + 40351, + 40383, + 40415, + 40447, + 40479, + 40511, + 40543, + 40575, + 40607, + 40639, + 40671, + 40703, + 40735, + 40767, + 40799, + 40831, + 40863, + 40895, + 40927, + 40959, + 40991, + 41023, + 41055, + 41087, + 41119, + 41151, + 41183, + 41215, + 41247, + 41279, + 41311, + 41343, + 41375, + 41407, + 41439, + 41471, + 41503, + 41535, + 41567, + 41599, + 41631, + 41663, + 41695, + 41727, + 41759, + 41791, + 41823, + 41855, + 41887, + 41919, + 41951, + 41983, + 42015, + 42047, + 42079, + 42111, + 42143, + 42175, + 42207, + 42239, + 42271, + 42303, + 42335, + 42367, + 42399, + 42431, + 42463, + 42495, + 42527, + 42559, + 42591, + 42623, + 42655, + 42687, + 42719, + 42751, + 42783, + 42815, + 42847, + 42879, + 42911, + 42943, + 42975, + 43007, + 43039, + 43071, + 43103, + 43135, + 43167, + 43199, + 43231, + 43263, + 43295, + 43327, + 43359, + 43391, + 43423, + 43455, + 43487, + 43519, + 43551, + 43583, + 43615, + 43647, + 43679, + 43711, + 43743, + 43775, + 43807, + 43839, + 43871, + 43903, + 43935, + 43967, + 43999, + 44031, + 44063, + 44095, + 44127, + 44159, + 44191, + 44223, + 44255, + 44287, + 44319, + 44351, + 44383, + 44415, + 44447, + 44479, + 44511, + 44543, + 44575, + 44607, + 44639, + 44671, + 44703, + 44735, + 44767, + 44799, + 44831, + 44863, + 44895, + 44927, + 44959, + 44991, + 45023, + 45055, + 45087, + 45119, + 45151, + 45183, + 45215, + 45247, + 45279, + 45311, + 45343, + 45375, + 45407, + 45439, + 45471, + 45503, + 45535, + 45567, + 45599, + 45631, + 45663, + 45695, + 45727, + 45759, + 45791, + 45823, + 45855, + 45887, + 45919, + 45951, + 45983, + 46015, + 46047, + 46079, + 46111, + 46143, + 46175, + 46207, + 46239, + 46271, + 46303, + 46335, + 46367, + 46399, + 46431, + 46463, + 46495, + 46527, + 46559, + 46591, + 46623, + 46655, + 46687, + 46719, + 46751, + 46783, + 46815, + 46847, + 46879, + 46911, + 46943, + 46975, + 47007, + 47039, + 47071, + 47103, + 47135, + 47167, + 47199, + 47231, + 47263, + 47295, + 47327, + 47359, + 47391, + 47423, + 47455, + 47487, + 47519, + 47551, + 47583, + 47615, + 47647, + 47679, + 47711, + 47743, + 47775, + 47807, + 47839, + 47871, + 47903, + 47935, + 47967, + 47999, + 48031, + 48063, + 48095, + 48127, + 48159, + 48191, + 48223, + 48255, + 48287, + 48319, + 48351, + 48383, + 48415, + 48447, + 48479, + 48511, + 48543, + 48575, + 48607, + 48639, + 48671, + 48703, + 48735, + 48767, + 48799, + 48831, + 48863, + 48895, + 48927, + 48959, + 48991, + 49023, + 49055, + 49087, + 49119, + 49151, + 49183, + 49215, + 49247, + 49279, + 49311, + 49343, + 49375, + 49407, + 49439, + 49471, + 49503, + 49535, + 49567, + 49599, + 49631, + 49663, + 49695, + 49727, + 49759, + 49791, + 49823, + 49855, + 49887, + 49919, + 49951, + 49983, + 50015, + 50047, + 50079, + 50111, + 50143, + 50175, + 50207, + 50239, + 50271, + 50303, + 50335, + 50367, + 50399, + 50431, + 50463, + 50495, + 50527, + 50559, + 50591, + 50623, + 50655, + 50687, + 50719, + 50751, + 50783, + 50815, + 50847, + 50879, + 50911, + 50943, + 50975, + 51007, + 51039, + 51071, + 51103, + 51135, + 51167, + 51199, + 51231, + 51263, + 51295, + 51327, + 51359, + 51391, + 51423, + 51455, + 51487, + 51519, + 51551, + 51583, + 51615, + 51647, + 51679, + 51711, + 51743, + 51775, + 51807, + 51839, + 51871, + 51903, + 51935, + 51967, + 51999, + 52031, + 52063, + 52095, + 52127, + 52159, + 52191, + 52223, + 52255, + 52287, + 52319, + 52351, + 52383, + 52415, + 52447, + 52479, + 52511, + 52543, + 52575, + 52607, + 52639, + 52671, + 52703, + 52735, + 52767, + 52799, + 52831, + 52863, + 52895, + 52927, + 52959, + 52991, + 53023, + 53055, + 53087, + 53119, + 53151, + 53183, + 53215, + 53247, + 53279, + 53311, + 53343, + 53375, + 53407, + 53439, + 53471, + 53503, + 53535, + 53567, + 53599, + 53631, + 53663, + 53695, + 53727, + 53759, + 53791, + 53823, + 53855, + 53887, + 53919, + 53951, + 53983, + 54015, + 54047, + 54079, + 54111, + 54143, + 54175, + 54207, + 54239, + 54271, + 54303, + 54335, + 54367, + 54399, + 54431, + 54463, + 54495, + 54527, + 54559, + 54591, + 54623, + 54655, + 54687, + 54719, + 54751, + 54783, + 54815, + 54847, + 54879, + 54911, + 54943, + 54975, + 55007, + 55039, + 55071, + 55103, + 55135, + 55167, + 55199, + 55231, + 55263, + 55295, + 55327, + 55359, + 55391, + 55423, + 55455, + 55487, + 55519, + 55551, + 55583, + 55615, + 55647, + 55679, + 55711, + 55743, + 55775, + 55807, + 55839, + 55871, + 55903, + 55935, + 55967, + 55999, + 56031, + 56063, + 56095, + 56127, + 56159, + 56191, + 56223, + 56255, + 56287, + 56319, + 56351, + 56383, + 56415, + 56447, + 56479, + 56511, + 56543, + 56575, + 56607, + 56639, + 56671, + 56703, + 56735, + 56767, + 56799, + 56831, + 56863, + 56895, + 56927, + 56959, + 56991, + 57023, + 57055, + 57087, + 57119, + 57151, + 57183, + 57215, + 57247, + 57279, + 57311, + 57343, + 57375, + 57407, + 57439, + 57471, + 57503, + 57535, + 57567, + 57599, + 57631, + 57663, + 57695, + 57727, + 57759, + 57791, + 57823, + 57855, + 57887, + 57919, + 57951, + 57983, + 58015, + 58047, + 58079, + 58111, + 58143, + 58175, + 58207, + 58239, + 58271, + 58303, + 58335, + 58367, + 58399, + 58431, + 58463, + 58495, + 58527, + 58559, + 58591, + 58623, + 58655, + 58687, + 58719, + 58751, + 58783, + 58815, + 58847, + 58879, + 58911, + 58943, + 58975, + 59007, + 59039, + 59071, + 59103, + 59135, + 59167, + 59199, + 59231, + 59263, + 59295, + 59327, + 59359, + 59391, + 59423, + 59455, + 59487, + 59519, + 59551, + 59583, + 59615, + 59647, + 59679, + 59711, + 59743, + 59775, + 59807, + 59839, + 59871, + 59903, + 59935, + 59967, + 59999, + 60031, + 60063, + 60095, + 60127, + 60159, + 60191, + 60223, + 60255, + 60287, + 60319, + 60351, + 60383, + 60415, + 60447, + 60479, + 60511, + 60543, + 60575, + 60607, + 60639, + 60671, + 60703, + 60735, + 60767, + 60799, + 60831, + 60863, + 60895, + 60927, + 60959, + 60991, + 61023, + 61055, + 61087, + 61119, + 61151, + 61183, + 61215, + 61247, + 61279, + 61311, + 61343, + 61375, + 61407, + 61439, + 61471, + 61503, + 61535, + 61567, + 61599, + 61631, + 61663, + 61695, + 61727, + 61759, + 61791, + 61823, + 61855, + 61887, + 61919, + 61951, + 61983, + 62015, + 62047, + 62079, + 62111, + 62143, + 62175, + 62207, + 62239, + 62271, + 62303, + 62335, + 62367, + 62399, + 62431, + 62463, + 62495, + 62527, + 62559, + 62591, + 62623, + 62655, + 62687, + 62719, + 62751, + 62783, + 62815, + 62847, + 62879, + 62911, + 62943, + 62975, + 63007, + 63039, + 63071, + 63103, + 63135, + 63167, + 63199, + 63231, + 63263, + 63295, + 63327, + 63359, + 63391, + 63423, + 63455, + 63487, + 63519, + 63551, + 63583, + 63615, + 63647, + 63679, + 63711, + 63743, + 63775, + 63807, + 63839, + 63871, + 63903, + 63935, + 63967, + 63999, + 64031, + 64063, + 64095, + 64127, + 64159, + 64191, + 64223, + 64255, + 64287, + 64319, + 64351, + 64383, + 64415, + 64447, + 64479, + 64511, + 64543, + 64575, + 64607, + 64639, + 64671, + 64703, + 64735, + 64767, + 64799, + 64831, + 64863, + 64895, + 64927, + 64959, + 64991, + 65023, + 65055, + 65087, + 65119, + 65151, + 65183, + 65215, + 65247, + 65279, + 65311, + 65343, + 65375, + 65407, + 65439, + 65471, + 65503, + 65535, + 65567, + 65599, + 65631, + 65663, + 65695, + 65727, + 65759, + 65791, + 65823, + 65855, + 65887, + 65919, + 65951, + 65983, + 66015, + 66047, + 66079, + 66111, + 66143, + 66175, + 66207, + 66239, + 66271, + 66303, + 66335, + 66367, + 66399, + 66431, + 66463, + 66495, + 66527, + 66559, + 66591, + 66623, + 66655, + 66687, + 66719, + 66751, + 66783, + 66815, + 66847, + 66879, + 66911, + 66943, + 66975, + 67007, + 67039, + 67071, + 67103, + 67135, + 67167, + 67199, + 67231, + 67263, + 67295, + 67327, + 67359, + 67391, + 67423, + 67455, + 67487, + 67519, + 67551, + 67583, + 67615, + 67647, + 67679, + 67711, + 67743, + 67775, + 67807, + 67839, + 67871, + 67903, + 67935, + 67967, + 67999, + 68031, + 68063, + 68095, + 68127, + 68159, + 68191, + 68223, + 68255, + 68287, + 68319, + 68351, + 68383, + 68415, + 68447, + 68479, + 68511, + 68543, + 68575, + 68607, + 68639, + 68671, + 68703, + 68735, + 68767, + 68799, + 68831, + 68863, + 68895, + 68927, + 68959, + 68991, + 69023, + 69055, + 69087, + 69119, + 69151, + 69183, + 69215, + 69247, + 69279, + 69311, + 69343, + 69375, + 69407, + 69439, + 69471, + 69503, + 69535, + 69567, + 69599, + 69631, + 69663, + 69695, + 69727, + 69759, + 69791, + 69823, + 69855, + 69887, + 69919, + 69951, + 69983, + 70015, + 70047, + 70079, + 70111, + 70143, + 70175, + 70207, + 70239, + 70271, + 70303, + 70335, + 70367, + 70399, + 70431, + 70463, + 70495, + 70527, + 70559, + 70591, + 70623, + 70655, + 70687, + 70719, + 70751, + 70783, + 70815, + 70847, + 70879, + 70911, + 70943, + 70975, + 71007, + 71039, + 71071, + 71103, + 71135, + 71167, + 71199, + 71231, + 71263, + 71295, + 71327, + 71359, + 71391, + 71423, + 71455, + 71487, + 71519, + 71551, + 71583, + 71615, + 71647, + 71679, + 71711, + 71743, + 71775, + 71807, + 71839, + 71871, + 71903, + 71935, + 71967, + 71999, + 72031, + 72063, + 72095, + 72127, + 72159, + 72191, + 72223, + 72255, + 72287, + 72319, + 72351, + 72383, + 72415, + 72447, + 72479, + 72511, + 72543, + 72575, + 72607, + 72639, + 72671, + 72703, + 72735, + 72767, + 72799, + 72831, + 72863, + 72895, + 72927, + 72959, + 72991, + 73023, + 73055, + 73087, + 73119, + 73151, + 73183, + 73215, + 73247, + 73279, + 73311, + 73343, + 73375, + 73407, + 73439, + 73471, + 73503, + 73535, + 73567, + 73599, + 73631, + 73663, + 73695, + 73727, + 73759, + 73791, + 73823, + 73855, + 73887, + 73919, + 73951, + 73983, + 74015, + 74047, + 74079, + 74111, + 74143, + 74175, + 74207, + 74239, + 74271, + 74303, + 74335, + 74367, + 74399, + 74431, + 74463, + 74495, + 74527, + 74559, + 74591, + 74623, + 74655, + 74687, + 74719, + 74751, + 74783, + 74815, + 74847, + 74879, + 74911, + 74943, + 74975, + 75007, + 75039, + 75071, + 75103, + 75135, + 75167, + 75199, + 75231, + 75263, + 75295, + 75327, + 75359, + 75391, + 75423, + 75455, + 75487, + 75519, + 75551, + 75583, + 75615, + 75647, + 75679, + 75711, + 75743, + 75775, + 75807, + 75839, + 75871, + 75903, + 75935, + 75967, + 75999, + 76031, + 76063, + 76095, + 76127, + 76159, + 76191, + 76223, + 76255, + 76287, + 76319, + 76351, + 76383, + 76415, + 76447, + 76479, + 76511, + 76543, + 76575, + 76607, + 76639, + 76671, + 76703, + 76735, + 76767, + 76799, + 76831, + 76863, + 76895, + 76927, + 76959, + 76991, + 77023, + 77055, + 77087, + 77119, + 77151, + 77183, + 77215, + 77247, + 77279, + 77311, + 77343, + 77375, + 77407, + 77439, + 77471, + 77503, + 77535, + 77567, + 77599, + 77631, + 77663, + 77695, + 77727, + 77759, + 77791, + 77823, + 77855, + 77887, + 77919, + 77951, + 77983, + 78015, + 78047, + 78079, + 78111, + 78143, + 78175, + 78207, + 78239, + 78271, + 78303, + 78335, + 78367, + 78399, + 78431, + 78463, + 78495, + 78527, + 78559, + 78591, + 78623, + 78655, + 78687, + 78719, + 78751, + 78783, + 78815, + 78847, + 78879, + 78911, + 78943, + 78975, + 79007, + 79039, + 79071, + 79103, + 79135, + 79167, + 79199, + 79231, + 79263, + 79295, + 79327, + 79359, + 79391, + 79423, + 79455, + 79487, + 79519, + 79551, + 79583, + 79615, + 79647, + 79679, + 79711, + 79743, + 79775, + 79807, + 79839, + 79871, + 79903, + 79935, + 79967, + 79999, + 80031, + 80063, + 80095, + 80127, + 80159, + 80191, + 80223, + 80255, + 80287, + 80319, + 80351, + 80383, + 80415, + 80447, + 80479, + 80511, + 80543, + 80575, + 80607, + 80639, + 80671, + 80703, + 80735, + 80767, + 80799, + 80831, + 80863, + 80895, + 80927, + 80959, + 80991, + 81023, + 81055, + 81087, + 81119, + 81151, + 81183, + 81215, + 81247, + 81279, + 81311, + 81343, + 81375, + 81407, + 81439, + 81471, + 81503, + 81535, + 81567, + 81599, + 81631, + 81663, + 81695, + 81727, + 81759, + 81791, + 81823, + 81855, + 81887, + 81919, + 81951, + 81983, + 82015, + 82047, + 82079, + 82111, + 82143, + 82175, + 82207, + 82239, + 82271, + 82303, + 82335, + 82367, + 82399, + 82431, + 82463, + 82495, + 82527, + 82559, + 82591, + 82623, + 82655, + 82687, + 82719, + 82751, + 82783, + 82815, + 82847, + 82879, + 82911, + 82943, + 82975, + 83007, + 83039, + 83071, + 83103, + 83135, + 83167, + 83199, + 83231, + 83263, + 83295, + 83327, + 83359, + 83391, + 83423, + 83455, + 83487, + 83519, + 83551, + 83583, + 83615, + 83647, + 83679, + 83711, + 83743, + 83775, + 83807, + 83839, + 83871, + 83903, + 83935, + 83967, + 83999, + 84031, + 84063, + 84095, + 84127, + 84159, + 84191, + 84223, + 84255, + 84287, + 84319, + 84351, + 84383, + 84415, + 84447, + 84479, + 84511, + 84543, + 84575, + 84607, + 84639, + 84671, + 84703, + 84735, + 84767, + 84799, + 84831, + 84863, + 84895, + 84927, + 84959, + 84991, + 85023, + 85055, + 85087, + 85119, + 85151, + 85183, + 85215, + 85247, + 85279, + 85311, + 85343, + 85375, + 85407, + 85439, + 85471, + 85503, + 85535, + 85567, + 85599, + 85631, + 85663, + 85695, + 85727, + 85759, + 85791, + 85823, + 85855, + 85887, + 85919, + 85951, + 85983, + 86015, + 86047, + 86079, + 86111, + 86143, + 86175, + 86207, + 86239, + 86271, + 86303, + 86335, + 86367, + 86399, + 86431, + 86463, + 86495, + 86527, + 86559, + 86591, + 86623, + 86655, + 86687, + 86719, + 86751, + 86783, + 86815, + 86847, + 86879, + 86911, + 86943, + 86975, + 87007, + 87039, + 87071, + 87103, + 87135, + 87167, + 87199, + 87231, + 87263, + 87295, + 87327, + 87359, + 87391, + 87423, + 87455, + 87487, + 87519, + 87551, + 87583, + 87615, + 87647, + 87679, + 87711, + 87743, + 87775, + 87807, + 87839, + 87871, + 87903, + 87935, + 87967, + 87999, + 88031, + 88063, + 88095, + 88127, + 88159, + 88191, + 88223, + 88255, + 88287, + 88319, + 88351, + 88383, + 88415, + 88447, + 88479, + 88511, + 88543, + 88575, + 88607, + 88639, + 88671, + 88703, + 88735, + 88767, + 88799, + 88831, + 88863, + 88895, + 88927, + 88959, + 88991, + 89023, + 89055, + 89087, + 89119, + 89151, + 89183, + 89215, + 89247, + 89279, + 89311, + 89343, + 89375, + 89407, + 89439, + 89471, + 89503, + 89535, + 89567, + 89599, + 89631, + 89663, + 89695, + 89727, + 89759, + 89791, + 89823, + 89855, + 89887, + 89919, + 89951, + 89983, + 90015, + 90047, + 90079, + 90111, + 90143, + 90175, + 90207, + 90239, + 90271, + 90303, + 90335, + 90367, + 90399, + 90431, + 90463, + 90495, + 90527, + 90559, + 90591, + 90623, + 90655, + 90687, + 90719, + 90751, + 90783, + 90815, + 90847, + 90879, + 90911, + 90943, + 90975, + 91007, + 91039, + 91071, + 91103, + 91135, + 91167, + 91199, + 91231, + 91263, + 91295, + 91327, + 91359, + 91391, + 91423, + 91455, + 91487, + 91519, + 91551, + 91583, + 91615, + 91647, + 91679, + 91711, + 91743, + 91775, + 91807, + 91839, + 91871, + 91903, + 91935, + 91967, + 91999, + 92031, + 92063, + 92095, + 92127, + 92159, + 92191, + 92223, + 92255, + 92287, + 92319, + 92351, + 92383, + 92415, + 92447, + 92479, + 92511, + 92543, + 92575, + 92607, + 92639, + 92671, + 92703, + 92735, + 92767, + 92799, + 92831, + 92863, + 92895, + 92927, + 92959, + 92991, + 93023, + 93055, + 93087, + 93119, + 93151, + 93183, + 93215, + 93247, + 93279, + 93311, + 93343, + 93375, + 93407, + 93439, + 93471, + 93503, + 93535, + 93567, + 93599, + 93631, + 93663, + 93695, + 93727, + 93759, + 93791, + 93823, + 93855, + 93887, + 93919, + 93951, + 93983, + 94015, + 94047, + 94079, + 94111, + 94143, + 94175, + 94207, + 94239, + 94271, + 94303, + 94335, + 94367, + 94399, + 94431, + 94463, + 94495, + 94527, + 94559, + 94591, + 94623, + 94655, + 94687, + 94719, + 94751, + 94783, + 94815, + 94847, + 94879, + 94911, + 94943, + 94975, + 95007, + 95039, + 95071, + 95103, + 95135, + 95167, + 95199, + 95231, + 95263, + 95295, + 95327, + 95359, + 95391, + 95423, + 95455, + 95487, + 95519, + 95551, + 95583, + 95615, + 95647, + 95679, + 95711, + 95743, + 95775, + 95807, + 95839, + 95871, + 95903, + 95935, + 95967, + 95999, + 96031, + 96063, + 96095, + 96127, + 96159, + 96191, + 96223, + 96255, + 96287, + 96319, + 96351, + 96383, + 96415, + 96447, + 96479, + 96511, + 96543, + 96575, + 96607, + 96639, + 96671, + 96703, + 96735, + 96767, + 96799, + 96831, + 96863, + 96895, + 96927, + 96959, + 96991, + 97023, + 97055, + 97087, + 97119, + 97151, + 97183, + 97215, + 97247, + 97279, + 97311, + 97343, + 97375, + 97407, + 97439, + 97471, + 97503, + 97535, + 97567, + 97599, + 97631, + 97663, + 97695, + 97727, + 97759, + 97791, + 97823, + 97855, + 97887, + 97919, + 97951, + 97983, + 98015, + 98047, + 98079, + 98111, + 98143, + 98175, + 98207, + 98239, + 98271, + 98303, + 98335, + 98367, + 98399, + 98431, + 98463, + 98495, + 98527, + 98559, + 98591, + 98623, + 98655, + 98687, + 98719, + 98751, + 98783, + 98815, + 98847, + 98879, + 98911, + 98943, + 98975, + 99007, + 99039, + 99071, + 99103, + 99135, + 99167, + 99199, + 99231, + 99263, + 99295, + 99327, + 99359, + 99391, + 99423, + 99455, + 99487, + 99519, + 99551, + 99583, + 99615, + 99647, + 99679, + 99711, + 99743, + 99775, + 99807, + 99839, + 99871, + 99903, + 99935, + 99967, + 99999, + 100031, + 100063, + 100095, + 100127, + 100159, + 100191, + 100223, + 100255, + 100287, + 100319, + 100351, + 100383, + 100415, + 100447, + 100479, + 100511, + 100543, + 100575, + 100607, + 100639, + 100671, + 100703, + 100735, + 100767, + 100799, + 100831, + 100863, + 100895, + 100927, + 100959, + 100991, + 101023, + 101055, + 101087, + 101119, + 101151, + 101183, + 101215, + 101247, + 101279, + 101311, + 101343, + 101375, + 101407, + 101439, + 101471, + 101503, + 101535, + 101567, + 101599, + 101631, + 101663, + 101695, + 101727, + 101759, + 101791, + 101823, + 101855, + 101887, + 101919, + 101951, + 101983, + 102015, + 102047, + 102079, + 102111, + 102143, + 102175, + 102207, + 102239, + 102271, + 102303, + 102335, + 102367, + 102399, + 102431, + 102463, + 102495, + 102527, + 102559, + 102591, + 102623, + 102655, + 102687, + 102719, + 102751, + 102783, + 102815, + 102847, + 102879, + 102911, + 102943, + 102975, + 103007, + 103039, + 103071, + 103103, + 103135, + 103167, + 103199, + 103231, + 103263, + 103295, + 103327, + 103359, + 103391, + 103423, + 103455, + 103487, + 103519, + 103551, + 103583, + 103615, + 103647, + 103679, + 103711, + 103743, + 103775, + 103807, + 103839, + 103871, + 103903, + 103935, + 103967, + 103999, + 104031, + 104063, + 104095, + 104127, + 104159, + 104191, + 104223, + 104255, + 104287, + 104319, + 104351, + 104383, + 104415, + 104447, + 104479, + 104511, + 104543, + 104575, + 104607, + 104639, + 104671, + 104703, + 104735, + 104767, + 104799, + 104831, + 104863, + 104895, + 104927, + 104959, + 104991, + 105023, + 105055, + 105087, + 105119, + 105151, + 105183, + 105215, + 105247, + 105279, + 105311, + 105343, + 105375, + 105407, + 105439, + 105471, + 105503, + 105535, + 105567, + 105599, + 105631, + 105663, + 105695, + 105727, + 105759, + 105791, + 105823, + 105855, + 105887, + 105919, + 105951, + 105983, + 106015, + 106047, + 106079, + 106111, + 106143, + 106175, + 106207, + 106239, + 106271, + 106303, + 106335, + 106367, + 106399, + 106431, + 106463, + 106495, + 106527, + 106559, + 106591, + 106623, + 106655, + 106687, + 106719, + 106751, + 106783, + 106815, + 106847, + 106879, + 106911, + 106943, + 106975, + 107007, + 107039, + 107071, + 107103, + 107135, + 107167, + 107199, + 107231, + 107263, + 107295, + 107327, + 107359, + 107391, + 107423, + 107455, + 107487, + 107519, + 107551, + 107583, + 107615, + 107647, + 107679, + 107711, + 107743, + 107775, + 107807, + 107839, + 107871, + 107903, + 107935, + 107967, + 107999, + 108031, + 108063, + 108095, + 108127, + 108159, + 108191, + 108223, + 108255, + 108287, + 108319, + 108351, + 108383, + 108415, + 108447, + 108479, + 108511, + 108543, + 108575, + 108607, + 108639, + 108671, + 108703, + 108735, + 108767, + 108799, + 108831, + 108863, + 108895, + 108927, + 108959, + 108991, + 109023, + 109055, + 109087, + 109119, + 109151, + 109183, + 109215, + 109247, + 109279, + 109311, + 109343, + 109375, + 109407, + 109439, + 109471, + 109503, + 109535, + 109567, + 109599, + 109631, + 109663, + 109695, + 109727, + 109759, + 109791, + 109823, + 109855, + 109887, + 109919, + 109951, + 109983, + 110015, + 110047, + 110079, + 110111, + 110143, + 110175, + 110207, + 110239, + 110271, + 110303, + 110335, + 110367, + 110399, + 110431, + 110463, + 110495, + 110527, + 110559, + 110591, + 110623, + 110655, + 110687, + 110719, + 110751, + 110783, + 110815, + 110847, + 110879, + 110911, + 110943, + 110975, + 111007, + 111039, + 111071, + 111103, + 111135, + 111167, + 111199, + 111231, + 111263, + 111295, + 111327, + 111359, + 111391, + 111423, + 111455, + 111487, + 111519, + 111551, + 111583, + 111615, + 111647, + 111679, + 111711, + 111743, + 111775, + 111807, + 111839, + 111871, + 111903, + 111935, + 111967, + 111999, + 112031, + 112063, + 112095, + 112127, + 112159, + 112191, + 112223, + 112255, + 112287, + 112319, + 112351, + 112383, + 112415, + 112447, + 112479, + 112511, + 112543, + 112575, + 112607, + 112639, + 112671, + 112703, + 112735, + 112767, + 112799, + 112831, + 112863, + 112895, + 112927, + 112959, + 112991, + 113023, + 113055, + 113087, + 113119, + 113151, + 113183, + 113215, + 113247, + 113279, + 113311, + 113343, + 113375, + 113407, + 113439, + 113471, + 113503, + 113535, + 113567, + 113599, + 113631, + 113663, + 113695, + 113727, + 113759, + 113791, + 113823, + 113855, + 113887, + 113919, + 113951, + 113983, + 114015, + 114047, + 114079, + 114111, + 114143, + 114175, + 114207, + 114239, + 114271, + 114303, + 114335, + 114367, + 114399, + 114431, + 114463, + 114495, + 114527, + 114559, + 114591, + 114623, + 114655, + 114687, + 114719, + 114751, + 114783, + 114815, + 114847, + 114879, + 114911, + 114943, + 114975, + 115007, + 115039, + 115071, + 115103, + 115135, + 115167, + 115199, + 115231, + 115263, + 115295, + 115327, + 115359, + 115391, + 115423, + 115455, + 115487, + 115519, + 115551, + 115583, + 115615, + 115647, + 115679, + 115711, + 115743, + 115775, + 115807, + 115839, + 115871, + 115903, + 115935, + 115967, + 115999, + 116031, + 116063, + 116095, + 116127, + 116159, + 116191, + 116223, + 116255, + 116287, + 116319, + 116351, + 116383, + 116415, + 116447, + 116479, + 116511, + 116543, + 116575, + 116607, + 116639, + 116671, + 116703, + 116735, + 116767, + 116799, + 116831, + 116863, + 116895, + 116927, + 116959, + 116991, + 117023, + 117055, + 117087, + 117119, + 117151, + 117183, + 117215, + 117247, + 117279, + 117311, + 117343, + 117375, + 117407, + 117439, + 117471, + 117503, + 117535, + 117567, + 117599, + 117631, + 117663, + 117695, + 117727, + 117759, + 117791, + 117823, + 117855, + 117887, + 117919, + 117951, + 117983, + 118015, + 118047, + 118079, + 118111, + 118143, + 118175, + 118207, + 118239, + 118271, + 118303, + 118335, + 118367, + 118399, + 118431, + 118463, + 118495, + 118527, + 118559, + 118591, + 118623, + 118655, + 118687, + 118719, + 118751, + 118783, + 118815, + 118847, + 118879, + 118911, + 118943, + 118975, + 119007, + 119039, + 119071, + 119103, + 119135, + 119167, + 119199, + 119231, + 119263, + 119295, + 119327, + 119359, + 119391, + 119423, + 119455, + 119487, + 119519, + 119551, + 119583, + 119615, + 119647, + 119679, + 119711, + 119743, + 119775, + 119807, + 119839, + 119871, + 119903, + 119935, + 119967, + 119999, + 120031, + 120063, + 120095, + 120127, + 120159, + 120191, + 120223, + 120255, + 120287, + 120319, + 120351, + 120383, + 120415, + 120447, + 120479, + 120511, + 120543, + 120575, + 120607, + 120639, + 120671, + 120703, + 120735, + 120767, + 120799, + 120831, + 120863, + 120895, + 120927, + 120959, + 120991, + 121023, + 121055, + 121087, + 121119, + 121151, + 121183, + 121215, + 121247, + 121279, + 121311, + 121343, + 121375, + 121407, + 121439, + 121471, + 121503, + 121535, + 121567, + 121599, + 121631, + 121663, + 121695, + 121727, + 121759, + 121791, + 121823, + 121855, + 121887, + 121919, + 121951, + 121983, + 122015, + 122047, + 122079, + 122111, + 122143, + 122175, + 122207, + 122239, + 122271, + 122303, + 122335, + 122367, + 122399, + 122431, + 122463, + 122495, + 122527, + 122559, + 122591, + 122623, + 122655, + 122687, + 122719, + 122751, + 122783, + 122815, + 122847, + 122879, + 122911, + 122943, + 122975, + 123007, + 123039, + 123071, + 123103, + 123135, + 123167, + 123199, + 123231, + 123263, + 123295, + 123327, + 123359, + 123391, + 123423, + 123455, + 123487, + 123519, + 123551, + 123583, + 123615, + 123647, + 123679, + 123711, + 123743, + 123775, + 123807, + 123839, + 123871, + 123903, + 123935, + 123967, + 123999, + 124031, + 124063, + 124095, + 124127, + 124159, + 124191, + 124223, + 124255, + 124287, + 124319, + 124351, + 124383, + 124415, + 124447, + 124479, + 124511, + 124543, + 124575, + 124607, + 124639, + 124671, + 124703, + 124735, + 124767, + 124799, + 124831, + 124863, + 124895, + 124927, + 124959, + 124991, + 125023, + 125055, + 125087, + 125119, + 125151, + 125183, + 125215, + 125247, + 125279, + 125311, + 125343, + 125375, + 125407, + 125439, + 125471, + 125503, + 125535, + 125567, + 125599, + 125631, + 125663, + 125695, + 125727, + 125759, + 125791, + 125823, + 125855, + 125887, + 125919, + 125951, + 125983, + 126015, + 126047, + 126079, + 126111, + 126143, + 126175, + 126207, + 126239, + 126271, + 126303, + 126335, + 126367, + 126399, + 126431, + 126463, + 126495, + 126527, + 126559, + 126591, + 126623, + 126655, + 126687, + 126719, + 126751, + 126783, + 126815, + 126847, + 126879, + 126911, + 126943, + 126975, + 127007, + 127039, + 127071, + 127103, + 127135, + 127167, + 127199, + 127231, + 127263, + 127295, + 127327, + 127359, + 127391, + 127423, + 127455, + 127487, + 127519, + 127551, + 127583, + 127615, + 127647, + 127679, + 127711, + 127743, + 127775, + 127807, + 127839, + 127871, + 127903, + 127935, + 127967, + 127999, + 128031, + 128063, + 128095, + 128127, + 128159, + 128191, + 128223, + 128255, + 128287, + 128319, + 128351, + 128383, + 128415, + 128447, + 128479, + 128511, + 128543, + 128575, + 128607, + 128639, + 128671, + 128703, + 128735, + 128767, + 128799, + 128831, + 128863, + 128895, + 128927, + 128959, + 128991, + 129023, + 129055, + 129087, + 129119, + 129151, + 129183, + 129215, + 129247, + 129279, + 129311, + 129343, + 129375, + 129407, + 129439, + 129471, + 129503, + 129535, + 129567, + 129599, + 129631, + 129663, + 129695, + 129727, + 129759, + 129791, + 129823, + 129855, + 129887, + 129919, + 129951, + 129983, + 130015, + 130047, + 130079, + 130111, + 130143, + 130175, + 130207, + 130239, + 130271, + 130303, + 130335, + 130367, + 130399, + 130431, + 130463, + 130495, + 130527, + 130559, + 130591, + 130623, + 130655, + 130687, + 130719, + 130751, + 130783, + 130815, + 130847, + 130879, + 130911, + 130943, + 130975, + 131007, + 131039, + 131071, + 131103, + 131135, + 131167, + 131199, + 131231, + 131263, + 131295, + 131327, + 131359, + 131391, + 131423, + 131455, + 131487, + 131519, + 131551, + 131583, + 131615, + 131647, + 131679, + 131711, + 131743, + 131775, + 131807, + 131839, + 131871, + 131903, + 131935, + 131967, + 131999, + 132031, + 132063, + 132095, + 132127, + 132159, + 132191, + 132223, + 132255, + 132287, + 132319, + 132351, + 132383, + 132415, + 132447, + 132479, + 132511, + 132543, + 132575, + 132607, + 132639, + 132671, + 132703, + 132735, + 132767, + 132799, + 132831, + 132863, + 132895, + 132927, + 132959, + 132991, + 133023, + 133055, + 133087, + 133119, + 133151, + 133183, + 133215, + 133247, + 133279, + 133311, + 133343, + 133375, + 133407, + 133439, + 133471, + 133503, + 133535, + 133567, + 133599, + 133631, + 133663, + 133695, + 133727, + 133759, + 133791, + 133823, + 133855, + 133887, + 133919, + 133951, + 133983, + 134015, + 134047, + 134079, + 134111, + 134143, + 134175, + 134207, + 134239, + 134271, + 134303, + 134335, + 134367, + 134399, + 134431, + 134463, + 134495, + 134527, + 134559, + 134591, + 134623, + 134655, + 134687, + 134719, + 134751, + 134783, + 134815, + 134847, + 134879, + 134911, + 134943, + 134975, + 135007, + 135039, + 135071, + 135103, + 135135, + 135167, + 135199, + 135231, + 135263, + 135295, + 135327, + 135359, + 135391, + 135423, + 135455, + 135487, + 135519, + 135551, + 135583, + 135615, + 135647, + 135679, + 135711, + 135743, + 135775, + 135807, + 135839, + 135871, + 135903, + 135935, + 135967, + 135999, + 136031, + 136063, + 136095, + 136127, + 136159, + 136191, + 136223, + 136255, + 136287, + 136319, + 136351, + 136383, + 136415, + 136447, + 136479, + 136511, + 136543, + 136575, + 136607, + 136639, + 136671, + 136703, + 136735, + 136767, + 136799, + 136831, + 136863, + 136895, + 136927, + 136959, + 136991, + 137023, + 137055, + 137087, + 137119, + 137151, + 137183, + 137215, + 137247, + 137279, + 137311, + 137343, + 137375, + 137407, + 137439, + 137471, + 137503, + 137535, + 137567, + 137599, + 137631, + 137663, + 137695, + 137727, + 137759, + 137791, + 137823, + 137855, + 137887, + 137919, + 137951, + 137983, + 138015, + 138047, + 138079, + 138111, + 138143, + 138175, + 138207, + 138239, + 138271, + 138303, + 138335, + 138367, + 138399, + 138431, + 138463, + 138495, + 138527, + 138559, + 138591, + 138623, + 138655, + 138687, + 138719, + 138751, + 138783, + 138815, + 138847, + 138879, + 138911, + 138943, + 138975, + 139007, + 139039, + 139071, + 139103, + 139135, + 139167, + 139199, + 139231, + 139263, + 139295, + 139327, + 139359, + 139391, + 139423, + 139455, + 139487, + 139519, + 139551, + 139583, + 139615, + 139647, + 139679, + 139711, + 139743, + 139775, + 139807, + 139839, + 139871, + 139903, + 139935, + 139967, + 139999, + 140031, + 140063, + 140095, + 140127, + 140159, + 140191, + 140223, + 140255, + 140287, + 140319, + 140351, + 140383, + 140415, + 140447, + 140479, + 140511, + 140543, + 140575, + 140607, + 140639, + 140671, + 140703, + 140735, + 140767, + 140799, + 140831, + 140863, + 140895, + 140927, + 140959, + 140991, + 141023, + 141055, + 141087, + 141119, + 141151, + 141183, + 141215, + 141247, + 141279, + 141311, + 141343, + 141375, + 141407, + 141439, + 141471, + 141503, + 141535, + 141567, + 141599, + 141631, + 141663, + 141695, + 141727, + 141759, + 141791, + 141823, + 141855, + 141887, + 141919, + 141951, + 141983, + 142015, + 142047, + 142079, + 142111, + 142143, + 142175, + 142207, + 142239, + 142271, + 142303, + 142335, + 142367, + 142399, + 142431, + 142463, + 142495, + 142527, + 142559, + 142591, + 142623, + 142655, + 142687, + 142719, + 142751, + 142783, + 142815, + 142847, + 142879, + 142911, + 142943, + 142975, + 143007, + 143039, + 143071, + 143103, + 143135, + 143167, + 143199, + 143231, + 143263, + 143295, + 143327, + 143359, + 143391, + 143423, + 143455, + 143487, + 143519, + 143551, + 143583, + 143615, + 143647, + 143679, + 143711, + 143743, + 143775, + 143807, + 143839, + 143871, + 143903, + 143935, + 143967, + 143999, + 144031, + 144063, + 144095, + 144127, + 144159, + 144191, + 144223, + 144255, + 144287, + 144319, + 144351, + 144383, + 144415, + 144447, + 144479, + 144511, + 144543, + 144575, + 144607, + 144639, + 144671, + 144703, + 144735, + 144767, + 144799, + 144831, + 144863, + 144895, + 144927, + 144959, + 144991, + 145023, + 145055, + 145087, + 145119, + 145151, + 145183, + 145215, + 145247, + 145279, + 145311, + 145343, + 145375, + 145407, + 145439, + 145471, + 145503, + 145535, + 145567, + 145599, + 145631, + 145663, + 145695, + 145727, + 145759, + 145791, + 145823, + 145855, + 145887, + 145919, + 145951, + 145983, + 146015, + 146047, + 146079, + 146111, + 146143, + 146175, + 146207, + 146239, + 146271, + 146303, + 146335, + 146367, + 146399, + 146431, + 146463, + 146495, + 146527, + 146559, + 146591, + 146623, + 146655, + 146687, + 146719, + 146751, + 146783, + 146815, + 146847, + 146879, + 146911, + 146943, + 146975, + 147007, + 147039, + 147071, + 147103, + 147135, + 147167, + 147199, + 147231, + 147263, + 147295, + 147327, + 147359, + 147391, + 147423, + 147455, + 147487, + 147519, + 147551, + 147583, + 147615, + 147647, + 147679, + 147711, + 147743, + 147775, + 147807, + 147839, + 147871, + 147903, + 147935, + 147967, + 147999, + 148031, + 148063, + 148095, + 148127, + 148159, + 148191, + 148223, + 148255, + 148287, + 148319, + 148351, + 148383, + 148415, + 148447, + 148479, + 148511, + 148543, + 148575, + 148607, + 148639, + 148671, + 148703, + 148735, + 148767, + 148799, + 148831, + 148863, + 148895, + 148927, + 148959, + 148991, + 149023, + 149055, + 149087, + 149119, + 149151, + 149183, + 149215, + 149247, + 149279, + 149311, + 149343, + 149375, + 149407, + 149439, + 149471, + 149503, + 149535, + 149567, + 149599, + 149631, + 149663, + 149695, + 149727, + 149759, + 149791, + 149823, + 149855, + 149887, + 149919, + 149951, + 149983, + 150015, + 150047, + 150079, + 150111, + 150143, + 150175, + 150207, + 150239, + 150271, + 150303, + 150335, + 150367, + 150399, + 150431, + 150463, + 150495, + 150527, + 150559, + 150591, + 150623, + 150655, + 150687, + 150719, + 150751, + 150783, + 150815, + 150847, + 150879, + 150911, + 150943, + 150975, + 151007, + 151039, + 151071, + 151103, + 151135, + 151167, + 151199, + 151231, + 151263, + 151295, + 151327, + 151359, + 151391, + 151423, + 151455, + 151487, + 151519, + 151551, + 151583, + 151615, + 151647, + 151679, + 151711, + 151743, + 151775, + 151807, + 151839, + 151871, + 151903, + 151935, + 151967, + 151999, + 152031, + 152063, + 152095, + 152127, + 152159, + 152191, + 152223, + 152255, + 152287, + 152319, + 152351, + 152383, + 152415, + 152447, + 152479, + 152511, + 152543, + 152575, + 152607, + 152639, + 152671, + 152703, + 152735, + 152767, + 152799, + 152831, + 152863, + 152895, + 152927, + 152959, + 152991, + 153023, + 153055, + 153087, + 153119, + 153151, + 153183, + 153215, + 153247, + 153279, + 153311, + 153343, + 153375, + 153407, + 153439, + 153471, + 153503, + 153535, + 153567, + 153599, + 153631, + 153663, + 153695, + 153727, + 153759, + 153791, + 153823, + 153855, + 153887, + 153919, + 153951, + 153983, + 154015, + 154047, + 154079, + 154111, + 154143, + 154175, + 154207, + 154239, + 154271, + 154303, + 154335, + 154367, + 154399, + 154431, + 154463, + 154495, + 154527, + 154559, + 154591, + 154623, + 154655, + 154687, + 154719, + 154751, + 154783, + 154815, + 154847, + 154879, + 154911, + 154943, + 154975, + 155007, + 155039, + 155071, + 155103, + 155135, + 155167, + 155199, + 155231, + 155263, + 155295, + 155327, + 155359, + 155391, + 155423, + 155455, + 155487, + 155519, + 155551, + 155583, + 155615, + 155647, + 155679, + 155711, + 155743, + 155775, + 155807, + 155839, + 155871, + 155903, + 155935, + 155967, + 155999, + 156031, + 156063, + 156095, + 156127, + 156159, + 156191, + 156223, + 156255, + 156287, + 156319, + 156351, + 156383, + 156415, + 156447, + 156479, + 156511, + 156543, + 156575, + 156607, + 156639, + 156671, + 156703, + 156735, + 156767, + 156799, + 156831, + 156863, + 156895, + 156927, + 156959, + 156991, + 157023, + 157055, + 157087, + 157119, + 157151, + 157183, + 157215, + 157247, + 157279, + 157311, + 157343, + 157375, + 157407, + 157439, + 157471, + 157503, + 157535, + 157567, + 157599, + 157631, + 157663, + 157695, + 157727, + 157759, + 157791, + 157823, + 157855, + 157887, + 157919, + 157951, + 157983, + 158015, + 158047, + 158079, + 158111, + 158143, + 158175, + 158207, + 158239, + 158271, + 158303, + 158335, + 158367, + 158399, + 158431, + 158463, + 158495, + 158527, + 158559, + 158591, + 158623, + 158655, + 158687, + 158719, + 158751, + 158783, + 158815, + 158847, + 158879, + 158911, + 158943, + 158975, + 159007, + 159039, + 159071, + 159103, + 159135, + 159167, + 159199, + 159231, + 159263, + 159295, + 159327, + 159359, + 159391, + 159423, + 159455, + 159487, + 159519, + 159551, + 159583, + 159615, + 159647, + 159679, + 159711, + 159743, + 159775, + 159807, + 159839, + 159871, + 159903, + 159935, + 159967, + 159999, + 160031, + 160063, + 160095, + 160127, + 160159, + 160191, + 160223, + 160255, + 160287, + 160319, + 160351, + 160383, + 160415, + 160447, + 160479, + 160511, + 160543, + 160575, + 160607, + 160639, + 160671, + 160703, + 160735, + 160767, + 160799, + 160831, + 160863, + 160895, + 160927, + 160959, + 160991, + 161023, + 161055, + 161087, + 161119, + 161151, + 161183, + 161215, + 161247, + 161279, + 161311, + 161343, + 161375, + 161407, + 161439, + 161471, + 161503, + 161535, + 161567, + 161599, + 161631, + 161663, + 161695, + 161727, + 161759, + 161791, + 161823, + 161855, + 161887, + 161919, + 161951, + 161983, + 162015, + 162047, + 162079, + 162111, + 162143, + 162175, + 162207, + 162239, + 162271, + 162303, + 162335, + 162367, + 162399, + 162431, + 162463, + 162495, + 162527, + 162559, + 162591, + 162623, + 162655, + 162687, + 162719, + 162751, + 162783, + 162815, + 162847, + 162879, + 162911, + 162943, + 162975, + 163007, + 163039, + 163071, + 163103, + 163135, + 163167, + 163199, + 163231, + 163263, + 163295, + 163327, + 163359, + 163391, + 163423, + 163455, + 163487, + 163519, + 163551, + 163583, + 163615, + 163647, + 163679, + 163711, + 163743, + 163775, + 163807, + 163839, + 163871, + 163903, + 163935, + 163967, + 163999, + 164031, + 164063, + 164095, + 164127, + 164159, + 164191, + 164223, + 164255, + 164287, + 164319, + 164351, + 164383, + 164415, + 164447, + 164479, + 164511, + 164543, + 164575, + 164607, + 164639, + 164671, + 164703, + 164735, + 164767, + 164799, + 164831, + 164863, + 164895, + 164927, + 164959, + 164991, + 165023, + 165055, + 165087, + 165119, + 165151, + 165183, + 165215, + 165247, + 165279, + 165311, + 165343, + 165375, + 165407, + 165439, + 165471, + 165503, + 165535, + 165567, + 165599, + 165631, + 165663, + 165695, + 165727, + 165759, + 165791, + 165823, + 165855, + 165887, + 165919, + 165951, + 165983, + 166015, + 166047, + 166079, + 166111, + 166143, + 166175, + 166207, + 166239, + 166271, + 166303, + 166335, + 166367, + 166399, + 166431, + 166463, + 166495, + 166527, + 166559, + 166591, + 166623, + 166655, + 166687, + 166719, + 166751, + 166783, + 166815, + 166847, + 166879, + 166911, + 166943, + 166975, + 167007, + 167039, + 167071, + 167103, + 167135, + 167167, + 167199, + 167231, + 167263, + 167295, + 167327, + 167359, + 167391, + 167423, + 167455, + 167487, + 167519, + 167551, + 167583, + 167615, + 167647, + 167679, + 167711, + 167743, + 167775, + 167807, + 167839, + 167871, + 167903, + 167935, + 167967, + 167999, + 168031, + 168063, + 168095, + 168127, + 168159, + 168191, + 168223, + 168255, + 168287, + 168319, + 168351, + 168383, + 168415, + 168447, + 168479, + 168511, + 168543, + 168575, + 168607, + 168639, + 168671, + 168703, + 168735, + 168767, + 168799, + 168831, + 168863, + 168895, + 168927, + 168959, + 168991, + 169023, + 169055, + 169087, + 169119, + 169151, + 169183, + 169215, + 169247, + 169279, + 169311, + 169343, + 169375, + 169407, + 169439, + 169471, + 169503, + 169535, + 169567, + 169599, + 169631, + 169663, + 169695, + 169727, + 169759, + 169791, + 169823, + 169855, + 169887, + 169919, + 169951, + 169983, + 170015, + 170047, + 170079, + 170111, + 170143, + 170175, + 170207, + 170239, + 170271, + 170303, + 170335, + 170367, + 170399, + 170431, + 170463, + 170495, + 170527, + 170559, + 170591, + 170623, + 170655, + 170687, + 170719, + 170751, + 170783, + 170815, + 170847, + 170879, + 170911, + 170943, + 170975, + 171007, + 171039, + 171071, + 171103, + 171135, + 171167, + 171199, + 171231, + 171263, + 171295, + 171327, + 171359, + 171391, + 171423, + 171455, + 171487, + 171519, + 171551, + 171583, + 171615, + 171647, + 171679, + 171711, + 171743, + 171775, + 171807, + 171839, + 171871, + 171903, + 171935, + 171967, + 171999, + 172031, + 172063, + 172095, + 172127, + 172159, + 172191, + 172223, + 172255, + 172287, + 172319, + 172351, + 172383, + 172415, + 172447, + 172479, + 172511, + 172543, + 172575, + 172607, + 172639, + 172671, + 172703, + 172735, + 172767, + 172799, + 172831, + 172863, + 172895, + 172927, + 172959, + 172991, + 173023, + 173055, + 173087, + 173119, + 173151, + 173183, + 173215, + 173247, + 173279, + 173311, + 173343, + 173375, + 173407, + 173439, + 173471, + 173503, + 173535, + 173567, + 173599, + 173631, + 173663, + 173695, + 173727, + 173759, + 173791, + 173823, + 173855, + 173887, + 173919, + 173951, + 173983, + 174015, + 174047, + 174079, + 174111, + 174143, + 174175, + 174207, + 174239, + 174271, + 174303, + 174335, + 174367, + 174399, + 174431, + 174463, + 174495, + 174527, + 174559, + 174591, + 174623, + 174655, + 174687, + 174719, + 174751, + 174783, + 174815, + 174847, + 174879, + 174911, + 174943, + 174975, + 175007, + 175039, + 175071, + 175103, + 175135, + 175167, + 175199, + 175231, + 175263, + 175295, + 175327, + 175359, + 175391, + 175423, + 175455, + 175487, + 175519, + 175551, + 175583, + 175615, + 175647, + 175679, + 175711, + 175743, + 175775, + 175807, + 175839, + 175871, + 175903, + 175935, + 175967, + 175999, + 176031, + 176063, + 176095, + 176127, + 176159, + 176191, + 176223, + 176255, + 176287, + 176319, + 176351, + 176383, + 176415, + 176447, + 176479, + 176511, + 176543, + 176575, + 176607, + 176639, + 176671, + 176703, + 176735, + 176767, + 176799, + 176831, + 176863, + 176895, + 176927, + 176959, + 176991, + 177023, + 177055, + 177087, + 177119, + 177151, + 177183, + 177215, + 177247, + 177279, + 177311, + 177343, + 177375, + 177407, + 177439, + 177471, + 177503, + 177535, + 177567, + 177599, + 177631, + 177663, + 177695, + 177727, + 177759, + 177791, + 177823, + 177855, + 177887, + 177919, + 177951, + 177983, + 178015, + 178047, + 178079, + 178111, + 178143, + 178175, + 178207, + 178239, + 178271, + 178303, + 178335, + 178367, + 178399, + 178431, + 178463, + 178495, + 178527, + 178559, + 178591, + 178623, + 178655, + 178687, + 178719, + 178751, + 178783, + 178815, + 178847, + 178879, + 178911, + 178943, + 178975, + 179007, + 179039, + 179071, + 179103, + 179135, + 179167, + 179199, + 179231, + 179263, + 179295, + 179327, + 179359, + 179391, + 179423, + 179455, + 179487, + 179519, + 179551, + 179583, + 179615, + 179647, + 179679, + 179711, + 179743, + 179775, + 179807, + 179839, + 179871, + 179903, + 179935, + 179967, + 179999, + 180031, + 180063, + 180095, + 180127, + 180159, + 180191, + 180223, + 180255, + 180287, + 180319, + 180351, + 180383, + 180415, + 180447, + 180479, + 180511, + 180543, + 180575, + 180607, + 180639, + 180671, + 180703, + 180735, + 180767, + 180799, + 180831, + 180863, + 180895, + 180927, + 180959, + 180991, + 181023, + 181055, + 181087, + 181119, + 181151, + 181183, + 181215, + 181247, + 181279, + 181311, + 181343, + 181375, + 181407, + 181439, + 181471, + 181503, + 181535, + 181567, + 181599, + 181631, + 181663, + 181695, + 181727, + 181759, + 181791, + 181823, + 181855, + 181887, + 181919, + 181951, + 181983, + 182015, + 182047, + 182079, + 182111, + 182143, + 182175, + 182207, + 182239, + 182271, + 182303, + 182335, + 182367, + 182399, + 182431, + 182463, + 182495, + 182527, + 182559, + 182591, + 182623, + 182655, + 182687, + 182719, + 182751, + 182783, + 182815, + 182847, + 182879, + 182911, + 182943, + 182975, + 183007, + 183039, + 183071, + 183103, + 183135, + 183167, + 183199, + 183231, + 183263, + 183295, + 183327, + 183359, + 183391, + 183423, + 183455, + 183487, + 183519, + 183551, + 183583, + 183615, + 183647, + 183679, + 183711, + 183743, + 183775, + 183807, + 183839, + 183871, + 183903, + 183935, + 183967, + 183999, + 184031, + 184063, + 184095, + 184127, + 184159, + 184191, + 184223, + 184255, + 184287, + 184319, + 184351, + 184383, + 184415, + 184447, + 184479, + 184511, + 184543, + 184575, + 184607, + 184639, + 184671, + 184703, + 184735, + 184767, + 184799, + 184831, + 184863, + 184895, + 184927, + 184959, + 184991, + 185023, + 185055, + 185087, + 185119, + 185151, + 185183, + 185215, + 185247, + 185279, + 185311, + 185343, + 185375, + 185407, + 185439, + 185471, + 185503, + 185535, + 185567, + 185599, + 185631, + 185663, + 185695, + 185727, + 185759, + 185791, + 185823, + 185855, + 185887, + 185919, + 185951, + 185983, + 186015, + 186047, + 186079, + 186111, + 186143, + 186175, + 186207, + 186239, + 186271, + 186303, + 186335, + 186367, + 186399, + 186431, + 186463, + 186495, + 186527, + 186559, + 186591, + 186623, + 186655, + 186687, + 186719, + 186751, + 186783, + 186815, + 186847, + 186879, + 186911, + 186943, + 186975, + 187007, + 187039, + 187071, + 187103, + 187135, + 187167, + 187199, + 187231, + 187263, + 187295, + 187327, + 187359, + 187391, + 187423, + 187455, + 187487, + 187519, + 187551, + 187583, + 187615, + 187647, + 187679, + 187711, + 187743, + 187775, + 187807, + 187839, + 187871, + 187903, + 187935, + 187967, + 187999, + 188031, + 188063, + 188095, + 188127, + 188159, + 188191, + 188223, + 188255, + 188287, + 188319, + 188351, + 188383, + 188415, + 188447, + 188479, + 188511, + 188543, + 188575, + 188607, + 188639, + 188671, + 188703, + 188735, + 188767, + 188799, + 188831, + 188863, + 188895, + 188927, + 188959, + 188991, + 189023, + 189055, + 189087, + 189119, + 189151, + 189183, + 189215, + 189247, + 189279, + 189311, + 189343, + 189375, + 189407, + 189439, + 189471, + 189503, + 189535, + 189567, + 189599, + 189631, + 189663, + 189695, + 189727, + 189759, + 189791, + 189823, + 189855, + 189887, + 189919, + 189951, + 189983, + 190015, + 190047, + 190079, + 190111, + 190143, + 190175, + 190207, + 190239, + 190271, + 190303, + 190335, + 190367, + 190399, + 190431, + 190463, + 190495, + 190527, + 190559, + 190591, + 190623, + 190655, + 190687, + 190719, + 190751, + 190783, + 190815, + 190847, + 190879, + 190911, + 190943, + 190975, + 191007, + 191039, + 191071, + 191103, + 191135, + 191167, + 191199, + 191231, + 191263, + 191295, + 191327, + 191359, + 191391, + 191423, + 191455, + 191487, + 191519, + 191551, + 191583, + 191615, + 191647, + 191679, + 191711, + 191743, + 191775, + 191807, + 191839, + 191871, + 191903, + 191935, + 191967, + 191999, + 192031, + 192063, + 192095, + 192127, + 192159, + 192191, + 192223, + 192255, + 192287, + 192319, + 192351, + 192383, + 192415, + 192447, + 192479, + 192511, + 192543, + 192575, + 192607, + 192639, + 192671, + 192703, + 192735, + 192767, + 192799, + 192831, + 192863, + 192895, + 192927, + 192959, + 192991, + 193023, + 193055, + 193087, + 193119, + 193151, + 193183, + 193215, + 193247, + 193279, + 193311, + 193343, + 193375, + 193407, + 193439, + 193471, + 193503, + 193535, + 193567, + 193599, + 193631, + 193663, + 193695, + 193727, + 193759, + 193791, + 193823, + 193855, + 193887, + 193919, + 193951, + 193983, + 194015, + 194047, + 194079, + 194111, + 194143, + 194175, + 194207, + 194239, + 194271, + 194303, + 194335, + 194367, + 194399, + 194431, + 194463, + 194495, + 194527, + 194559, + 194591, + 194623, + 194655, + 194687, + 194719, + 194751, + 194783, + 194815, + 194847, + 194879, + 194911, + 194943, + 194975, + 195007, + 195039, + 195071, + 195103, + 195135, + 195167, + 195199, + 195231, + 195263, + 195295, + 195327, + 195359, + 195391, + 195423, + 195455, + 195487, + 195519, + 195551, + 195583, + 195615, + 195647, + 195679, + 195711, + 195743, + 195775, + 195807, + 195839, + 195871, + 195903, + 195935, + 195967, + 195999, + 196031, + 196063, + 196095, + 196127, + 196159, + 196191, + 196223, + 196255, + 196287, + 196319, + 196351, + 196383, + 196415, + 196447, + 196479, + 196511, + 196543, + 196575, + 196607, + 196639, + 196671, + 196703, + 196735, + 196767, + 196799, + 196831, + 196863, + 196895, + 196927, + 196959, + 196991, + 197023, + 197055, + 197087, + 197119, + 197151, + 197183, + 197215, + 197247, + 197279, + 197311, + 197343, + 197375, + 197407, + 197439, + 197471, + 197503, + 197535, + 197567, + 197599, + 197631, + 197663, + 197695, + 197727, + 197759, + 197791, + 197823, + 197855, + 197887, + 197919, + 197951, + 197983, + 198015, + 198047, + 198079, + 198111, + 198143, + 198175, + 198207, + 198239, + 198271, + 198303, + 198335, + 198367, + 198399, + 198431, + 198463, + 198495, + 198527, + 198559, + 198591, + 198623, + 198655, + 198687, + 198719, + 198751, + 198783, + 198815, + 198847, + 198879, + 198911, + 198943, + 198975, + 199007, + 199039, + 199071, + 199103, + 199135, + 199167, + 199199, + 199231, + 199263, + 199295, + 199327, + 199359, + 199391, + 199423, + 199455, + 199487, + 199519, + 199551, + 199583, + 199615, + 199647, + 199679, + 199711, + 199743, + 199775, + 199807, + 199839, + 199871, + 199903, + 199935, + 199967, + 199999, + 200031, + 200063, + 200095, + 200127, + 200159, + 200191, + 200223, + 200255, + 200287, + 200319, + 200351, + 200383, + 200415, + 200447, + 200479, + 200511, + 200543, + 200575, + 200607, + 200639, + 200671, + 200703, + 200735, + 200767, + 200799, + 200831, + 200863, + 200895, + 200927, + 200959, + 200991, + 201023, + 201055, + 201087, + 201119, + 201151, + 201183, + 201215, + 201247, + 201279, + 201311, + 201343, + 201375, + 201407, + 201439, + 201471, + 201503, + 201535, + 201567, + 201599, + 201631, + 201663, + 201695, + 201727, + 201759, + 201791, + 201823, + 201855, + 201887, + 201919, + 201951, + 201983, + 202015, + 202047, + 202079, + 202111, + 202143, + 202175, + 202207, + 202239, + 202271, + 202303, + 202335, + 202367, + 202399, + 202431, + 202463, + 202495, + 202527, + 202559, + 202591, + 202623, + 202655, + 202687, + 202719, + 202751, + 202783, + 202815, + 202847, + 202879, + 202911, + 202943, + 202975, + 203007, + 203039, + 203071, + 203103, + 203135, + 203167, + 203199, + 203231, + 203263, + 203295, + 203327, + 203359, + 203391, + 203423, + 203455, + 203487, + 203519, + 203551, + 203583, + 203615, + 203647, + 203679, + 203711, + 203743, + 203775, + 203807, + 203839, + 203871, + 203903, + 203935, + 203967, + 203999, + 204031, + 204063, + 204095, + 204127, + 204159, + 204191, + 204223, + 204255, + 204287, + 204319, + 204351, + 204383, + 204415, + 204447, + 204479, + 204511, + 204543, + 204575, + 204607, + 204639, + 204671, + 204703, + 204735, + 204767, + 204799, + 204831, + 204863, + 204895, + 204927, + 204959, + 204991, + 205023, + 205055, + 205087, + 205119, + 205151, + 205183, + 205215, + 205247, + 205279, + 205311, + 205343, + 205375, + 205407, + 205439, + 205471, + 205503, + 205535, + 205567, + 205599, + 205631, + 205663, + 205695, + 205727, + 205759, + 205791, + 205823, + 205855, + 205887, + 205919, + 205951, + 205983, + 206015, + 206047, + 206079, + 206111, + 206143, + 206175, + 206207, + 206239, + 206271, + 206303, + 206335, + 206367, + 206399, + 206431, + 206463, + 206495, + 206527, + 206559, + 206591, + 206623, + 206655, + 206687, + 206719, + 206751, + 206783, + 206815, + 206847, + 206879, + 206911, + 206943, + 206975, + 207007, + 207039, + 207071, + 207103, + 207135, + 207167, + 207199, + 207231, + 207263, + 207295, + 207327, + 207359, + 207391, + 207423, + 207455, + 207487, + 207519, + 207551, + 207583, + 207615, + 207647, + 207679, + 207711, + 207743, + 207775, + 207807, + 207839, + 207871, + 207903, + 207935, + 207967, + 207999, + 208031, + 208063, + 208095, + 208127, + 208159, + 208191, + 208223, + 208255, + 208287, + 208319, + 208351, + 208383, + 208415, + 208447, + 208479, + 208511, + 208543, + 208575, + 208607, + 208639, + 208671, + 208703, + 208735, + 208767, + 208799, + 208831, + 208863, + 208895, + 208927, + 208959, + 208991, + 209023, + 209055, + 209087, + 209119, + 209151, + 209183, + 209215, + 209247, + 209279, + 209311, + 209343, + 209375, + 209407, + 209439, + 209471, + 209503, + 209535, + 209567, + 209599, + 209631, + 209663, + 209695, + 209727, + 209759, + 209791, + 209823, + 209855, + 209887, + 209919, + 209951, + 209983, + 210015, + 210047, + 210079, + 210111, + 210143, + 210175, + 210207, + 210239, + 210271, + 210303, + 210335, + 210367, + 210399, + 210431, + 210463, + 210495, + 210527, + 210559, + 210591, + 210623, + 210655, + 210687, + 210719, + 210751, + 210783, + 210815, + 210847, + 210879, + 210911, + 210943, + 210975, + 211007, + 211039, + 211071, + 211103, + 211135, + 211167, + 211199, + 211231, + 211263, + 211295, + 211327, + 211359, + 211391, + 211423, + 211455, + 211487, + 211519, + 211551, + 211583, + 211615, + 211647, + 211679, + 211711, + 211743, + 211775, + 211807, + 211839, + 211871, + 211903, + 211935, + 211967, + 211999, + 212031, + 212063, + 212095, + 212127, + 212159, + 212191, + 212223, + 212255, + 212287, + 212319, + 212351, + 212383, + 212415, + 212447, + 212479, + 212511, + 212543, + 212575, + 212607, + 212639, + 212671, + 212703, + 212735, + 212767, + 212799, + 212831, + 212863, + 212895, + 212927, + 212959, + 212991, + 213023, + 213055, + 213087, + 213119, + 213151, + 213183, + 213215, + 213247, + 213279, + 213311, + 213343, + 213375, + 213407, + 213439, + 213471, + 213503, + 213535, + 213567, + 213599, + 213631, + 213663, + 213695, + 213727, + 213759, + 213791, + 213823, + 213855, + 213887, + 213919, + 213951, + 213983, + 214015, + 214047, + 214079, + 214111, + 214143, + 214175, + 214207, + 214239, + 214271, + 214303, + 214335, + 214367, + 214399, + 214431, + 214463, + 214495, + 214527, + 214559, + 214591, + 214623, + 214655, + 214687, + 214719, + 214751, + 214783, + 214815, + 214847, + 214879, + 214911, + 214943, + 214975, + 215007, + 215039, + 215071, + 215103, + 215135, + 215167, + 215199, + 215231, + 215263, + 215295, + 215327, + 215359, + 215391, + 215423, + 215455, + 215487, + 215519, + 215551, + 215583, + 215615, + 215647, + 215679, + 215711, + 215743, + 215775, + 215807, + 215839, + 215871, + 215903, + 215935, + 215967, + 215999, + 216031, + 216063, + 216095, + 216127, + 216159, + 216191, + 216223, + 216255, + 216287, + 216319, + 216351, + 216383, + 216415, + 216447, + 216479, + 216511, + 216543, + 216575, + 216607, + 216639, + 216671, + 216703, + 216735, + 216767, + 216799, + 216831, + 216863, + 216895, + 216927, + 216959, + 216991, + 217023, + 217055, + 217087, + 217119, + 217151, + 217183, + 217215, + 217247, + 217279, + 217311, + 217343, + 217375, + 217407, + 217439, + 217471, + 217503, + 217535, + 217567, + 217599, + 217631, + 217663, + 217695, + 217727, + 217759, + 217791, + 217823, + 217855, + 217887, + 217919, + 217951, + 217983, + 218015, + 218047, + 218079, + 218111, + 218143, + 218175, + 218207, + 218239, + 218271, + 218303, + 218335, + 218367, + 218399, + 218431, + 218463, + 218495, + 218527, + 218559, + 218591, + 218623, + 218655, + 218687, + 218719, + 218751, + 218783, + 218815, + 218847, + 218879, + 218911, + 218943, + 218975, + 219007, + 219039, + 219071, + 219103, + 219135, + 219167, + 219199, + 219231, + 219263, + 219295, + 219327, + 219359, + 219391, + 219423, + 219455, + 219487, + 219519, + 219551, + 219583, + 219615, + 219647, + 219679, + 219711, + 219743, + 219775, + 219807, + 219839, + 219871, + 219903, + 219935, + 219967, + 219999, + 220031, + 220063, + 220095, + 220127, + 220159, + 220191, + 220223, + 220255, + 220287, + 220319, + 220351, + 220383, + 220415, + 220447, + 220479, + 220511, + 220543, + 220575, + 220607, + 220639, + 220671, + 220703, + 220735, + 220767, + 220799, + 220831, + 220863, + 220895, + 220927, + 220959, + 220991, + 221023, + 221055, + 221087, + 221119, + 221151, + 221183, + 221215, + 221247, + 221279, + 221311, + 221343, + 221375, + 221407, + 221439, + 221471, + 221503, + 221535, + 221567, + 221599, + 221631, + 221663, + 221695, + 221727, + 221759, + 221791, + 221823, + 221855, + 221887, + 221919, + 221951, + 221983, + 222015, + 222047, + 222079, + 222111, + 222143, + 222175, + 222207, + 222239, + 222271, + 222303, + 222335, + 222367, + 222399, + 222431, + 222463, + 222495, + 222527, + 222559, + 222591, + 222623, + 222655, + 222687, + 222719, + 222751, + 222783, + 222815, + 222847, + 222879, + 222911, + 222943, + 222975, + 223007, + 223039, + 223071, + 223103, + 223135, + 223167, + 223199, + 223231, + 223263, + 223295, + 223327, + 223359, + 223391, + 223423, + 223455, + 223487, + 223519, + 223551, + 223583, + 223615, + 223647, + 223679, + 223711, + 223743, + 223775, + 223807, + 223839, + 223871, + 223903, + 223935, + 223967, + 223999, + 224031, + 224063, + 224095, + 224127, + 224159, + 224191, + 224223, + 224255, + 224287, + 224319, + 224351, + 224383, + 224415, + 224447, + 224479, + 224511, + 224543, + 224575, + 224607, + 224639, + 224671, + 224703, + 224735, + 224767, + 224799, + 224831, + 224863, + 224895, + 224927, + 224959, + 224991, + 225023, + 225055, + 225087, + 225119, + 225151, + 225183, + 225215, + 225247, + 225279, + 225311, + 225343, + 225375, + 225407, + 225439, + 225471, + 225503, + 225535, + 225567, + 225599, + 225631, + 225663, + 225695, + 225727, + 225759, + 225791, + 225823, + 225855, + 225887, + 225919, + 225951, + 225983, + 226015, + 226047, + 226079, + 226111, + 226143, + 226175, + 226207, + 226239, + 226271, + 226303, + 226335, + 226367, + 226399, + 226431, + 226463, + 226495, + 226527, + 226559, + 226591, + 226623, + 226655, + 226687, + 226719, + 226751, + 226783, + 226815, + 226847, + 226879, + 226911, + 226943, + 226975, + 227007, + 227039, + 227071, + 227103, + 227135, + 227167, + 227199, + 227231, + 227263, + 227295, + 227327, + 227359, + 227391, + 227423, + 227455, + 227487, + 227519, + 227551, + 227583, + 227615, + 227647, + 227679, + 227711, + 227743, + 227775, + 227807, + 227839, + 227871, + 227903, + 227935, + 227967, + 227999, + 228031, + 228063, + 228095, + 228127, + 228159, + 228191, + 228223, + 228255, + 228287, + 228319, + 228351, + 228383, + 228415, + 228447, + 228479, + 228511, + 228543, + 228575, + 228607, + 228639, + 228671, + 228703, + 228735, + 228767, + 228799, + 228831, + 228863, + 228895, + 228927, + 228959, + 228991, + 229023, + 229055, + 229087, + 229119, + 229151, + 229183, + 229215, + 229247, + 229279, + 229311, + 229343, + 229375, + 229407, + 229439, + 229471, + 229503, + 229535, + 229567, + 229599, + 229631, + 229663, + 229695, + 229727, + 229759, + 229791, + 229823, + 229855, + 229887, + 229919, + 229951, + 229983, + 230015, + 230047, + 230079, + 230111, + 230143, + 230175, + 230207, + 230239, + 230271, + 230303, + 230335, + 230367, + 230399, + 230431, + 230463, + 230495, + 230527, + 230559, + 230591, + 230623, + 230655, + 230687, + 230719, + 230751, + 230783, + 230815, + 230847, + 230879, + 230911, + 230943, + 230975, + 231007, + 231039, + 231071, + 231103, + 231135, + 231167, + 231199, + 231231, + 231263, + 231295, + 231327, + 231359, + 231391, + 231423, + 231455, + 231487, + 231519, + 231551, + 231583, + 231615, + 231647, + 231679, + 231711, + 231743, + 231775, + 231807, + 231839, + 231871, + 231903, + 231935, + 231967, + 231999, + 232031, + 232063, + 232095, + 232127, + 232159, + 232191, + 232223, + 232255, + 232287, + 232319, + 232351, + 232383, + 232415, + 232447, + 232479, + 232511, + 232543, + 232575, + 232607, + 232639, + 232671, + 232703, + 232735, + 232767, + 232799, + 232831, + 232863, + 232895, + 232927, + 232959, + 232991, + 233023, + 233055, + 233087, + 233119, + 233151, + 233183, + 233215, + 233247, + 233279, + 233311, + 233343, + 233375, + 233407, + 233439, + 233471, + 233503, + 233535, + 233567, + 233599, + 233631, + 233663, + 233695, + 233727, + 233759, + 233791, + 233823, + 233855, + 233887, + 233919, + 233951, + 233983, + 234015, + 234047, + 234079, + 234111, + 234143, + 234175, + 234207, + 234239, + 234271, + 234303, + 234335, + 234367, + 234399, + 234431, + 234463, + 234495, + 234527, + 234559, + 234591, + 234623, + 234655, + 234687, + 234719, + 234751, + 234783, + 234815, + 234847, + 234879, + 234911, + 234943, + 234975, + 235007, + 235039, + 235071, + 235103, + 235135, + 235167, + 235199, + 235231, + 235263, + 235295, + 235327, + 235359, + 235391, + 235423, + 235455, + 235487, + 235519, + 235551, + 235583, + 235615, + 235647, + 235679, + 235711, + 235743, + 235775, + 235807, + 235839, + 235871, + 235903, + 235935, + 235967, + 235999, + 236031, + 236063, + 236095, + 236127, + 236159, + 236191, + 236223, + 236255, + 236287, + 236319, + 236351, + 236383, + 236415, + 236447, + 236479, + 236511, + 236543, + 236575, + 236607, + 236639, + 236671, + 236703, + 236735, + 236767, + 236799, + 236831, + 236863, + 236895, + 236927, + 236959, + 236991, + 237023, + 237055, + 237087, + 237119, + 237151, + 237183, + 237215, + 237247, + 237279, + 237311, + 237343, + 237375, + 237407, + 237439, + 237471, + 237503, + 237535, + 237567, + 237599, + 237631, + 237663, + 237695, + 237727, + 237759, + 237791, + 237823, + 237855, + 237887, + 237919, + 237951, + 237983, + 238015, + 238047, + 238079, + 238111, + 238143, + 238175, + 238207, + 238239, + 238271, + 238303, + 238335, + 238367, + 238399, + 238431, + 238463, + 238495, + 238527, + 238559, + 238591, + 238623, + 238655, + 238687, + 238719, + 238751, + 238783, + 238815, + 238847, + 238879, + 238911, + 238943, + 238975, + 239007, + 239039, + 239071, + 239103, + 239135, + 239167, + 239199, + 239231, + 239263, + 239295, + 239327, + 239359, + 239391, + 239423, + 239455, + 239487, + 239519, + 239551, + 239583, + 239615, + 239647, + 239679, + 239711, + 239743, + 239775, + 239807, + 239839, + 239871, + 239903, + 239935, + 239967, + 239999, + 240031, + 240063, + 240095, + 240127, + 240159, + 240191, + 240223, + 240255, + 240287, + 240319, + 240351, + 240383, + 240415, + 240447, + 240479, + 240511, + 240543, + 240575, + 240607, + 240639, + 240671, + 240703, + 240735, + 240767, + 240799, + 240831, + 240863, + 240895, + 240927, + 240959, + 240991, + 241023, + 241055, + 241087, + 241119, + 241151, + 241183, + 241215, + 241247, + 241279, + 241311, + 241343, + 241375, + 241407, + 241439, + 241471, + 241503, + 241535, + 241567, + 241599, + 241631, + 241663, + 241695, + 241727, + 241759, + 241791, + 241823, + 241855, + 241887, + 241919, + 241951, + 241983, + 242015, + 242047, + 242079, + 242111, + 242143, + 242175, + 242207, + 242239, + 242271, + 242303, + 242335, + 242367, + 242399, + 242431, + 242463, + 242495, + 242527, + 242559, + 242591, + 242623, + 242655, + 242687, + 242719, + 242751, + 242783, + 242815, + 242847, + 242879, + 242911, + 242943, + 242975, + 243007, + 243039, + 243071, + 243103, + 243135, + 243167, + 243199, + 243231, + 243263, + 243295, + 243327, + 243359, + 243391, + 243423, + 243455, + 243487, + 243519, + 243551, + 243583, + 243615, + 243647, + 243679, + 243711, + 243743, + 243775, + 243807, + 243839, + 243871, + 243903, + 243935, + 243967, + 243999, + 244031, + 244063, + 244095, + 244127, + 244159, + 244191, + 244223, + 244255, + 244287, + 244319, + 244351, + 244383, + 244415, + 244447, + 244479, + 244511, + 244543, + 244575, + 244607, + 244639, + 244671, + 244703, + 244735, + 244767, + 244799, + 244831, + 244863, + 244895, + 244927, + 244959, + 244991, + 245023, + 245055, + 245087, + 245119, + 245151, + 245183, + 245215, + 245247, + 245279, + 245311, + 245343, + 245375, + 245407, + 245439, + 245471, + 245503, + 245535, + 245567, + 245599, + 245631, + 245663, + 245695, + 245727, + 245759, + 245791, + 245823, + 245855, + 245887, + 245919, + 245951, + 245983, + 246015, + 246047, + 246079, + 246111, + 246143, + 246175, + 246207, + 246239, + 246271, + 246303, + 246335, + 246367, + 246399, + 246431, + 246463, + 246495, + 246527, + 246559, + 246591, + 246623, + 246655, + 246687, + 246719, + 246751, + 246783, + 246815, + 246847, + 246879, + 246911, + 246943, + 246975, + 247007, + 247039, + 247071, + 247103, + 247135, + 247167, + 247199, + 247231, + 247263, + 247295, + 247327, + 247359, + 247391, + 247423, + 247455, + 247487, + 247519, + 247551, + 247583, + 247615, + 247647, + 247679, + 247711, + 247743, + 247775, + 247807, + 247839, + 247871, + 247903, + 247935, + 247967, + 247999, + 248031, + 248063, + 248095, + 248127, + 248159, + 248191, + 248223, + 248255, + 248287, + 248319, + 248351, + 248383, + 248415, + 248447, + 248479, + 248511, + 248543, + 248575, + 248607, + 248639, + 248671, + 248703, + 248735, + 248767, + 248799, + 248831, + 248863, + 248895, + 248927, + 248959, + 248991, + 249023, + 249055, + 249087, + 249119, + 249151, + 249183, + 249215, + 249247, + 249279, + 249311, + 249343, + 249375, + 249407, + 249439, + 249471, + 249503, + 249535, + 249567, + 249599, + 249631, + 249663, + 249695, + 249727, + 249759, + 249791, + 249823, + 249855, + 249887, + 249919, + 249951, + 249983, + 250015, + 250047, + 250079, + 250111, + 250143, + 250175, + 250207, + 250239, + 250271, + 250303, + 250335, + 250367, + 250399, + 250431, + 250463, + 250495, + 250527, + 250559, + 250591, + 250623, + 250655, + 250687, + 250719, + 250751, + 250783, + 250815, + 250847, + 250879, + 250911, + 250943, + 250975, + 251007, + 251039, + 251071, + 251103, + 251135, + 251167, + 251199, + 251231, + 251263, + 251295, + 251327, + 251359, + 251391, + 251423, + 251455, + 251487, + 251519, + 251551, + 251583, + 251615, + 251647, + 251679, + 251711, + 251743, + 251775, + 251807, + 251839, + 251871, + 251903, + 251935, + 251967, + 251999, + 252031, + 252063, + 252095, + 252127, + 252159, + 252191, + 252223, + 252255, + 252287, + 252319, + 252351, + 252383, + 252415, + 252447, + 252479, + 252511, + 252543, + 252575, + 252607, + 252639, + 252671, + 252703, + 252735, + 252767, + 252799, + 252831, + 252863, + 252895, + 252927, + 252959, + 252991, + 253023, + 253055, + 253087, + 253119, + 253151, + 253183, + 253215, + 253247, + 253279, + 253311, + 253343, + 253375, + 253407, + 253439, + 253471, + 253503, + 253535, + 253567, + 253599, + 253631, + 253663, + 253695, + 253727, + 253759, + 253791, + 253823, + 253855, + 253887, + 253919, + 253951, + 253983, + 254015, + 254047, + 254079, + 254111, + 254143, + 254175, + 254207, + 254239, + 254271, + 254303, + 254335, + 254367, + 254399, + 254431, + 254463, + 254495, + 254527, + 254559, + 254591, + 254623, + 254655, + 254687, + 254719, + 254751, + 254783, + 254815, + 254847, + 254879, + 254911, + 254943, + 254975, + 255007, + 255039, + 255071, + 255103, + 255135, + 255167, + 255199, + 255231, + 255263, + 255295, + 255327, + 255359, + 255391, + 255423, + 255455, + 255487, + 255519, + 255551, + 255583, + 255615, + 255647, + 255679, + 255711, + 255743, + 255775, + 255807, + 255839, + 255871, + 255903, + 255935, + 255967, + 255999, + 256031, + 256063, + 256095, + 256127, + 256159, + 256191, + 256223, + 256255, + 256287, + 256319, + 256351, + 256383, + 256415, + 256447, + 256479, + 256511, + 256543, + 256575, + 256607, + 256639, + 256671, + 256703, + 256735, + 256767, + 256799, + 256831, + 256863, + 256895, + 256927, + 256959, + 256991, + 257023, + 257055, + 257087, + 257119, + 257151, + 257183, + 257215, + 257247, + 257279, + 257311, + 257343, + 257375, + 257407, + 257439, + 257471, + 257503, + 257535, + 257567, + 257599, + 257631, + 257663, + 257695, + 257727, + 257759, + 257791, + 257823, + 257855, + 257887, + 257919, + 257951, + 257983, + 258015, + 258047, + 258079, + 258111, + 258143, + 258175, + 258207, + 258239, + 258271, + 258303, + 258335, + 258367, + 258399, + 258431, + 258463, + 258495, + 258527, + 258559, + 258591, + 258623, + 258655, + 258687, + 258719, + 258751, + 258783, + 258815, + 258847, + 258879, + 258911, + 258943, + 258975, + 259007, + 259039, + 259071, + 259103, + 259135, + 259167, + 259199, + 259231, + 259263, + 259295, + 259327, + 259359, + 259391, + 259423, + 259455, + 259487, + 259519, + 259551, + 259583, + 259615, + 259647, + 259679, + 259711, + 259743, + 259775, + 259807, + 259839, + 259871, + 259903, + 259935, + 259967, + 259999, + 260031, + 260063, + 260095, + 260127, + 260159, + 260191, + 260223, + 260255, + 260287, + 260319, + 260351, + 260383, + 260415, + 260447, + 260479, + 260511, + 260543, + 260575, + 260607, + 260639, + 260671, + 260703, + 260735, + 260767, + 260799, + 260831, + 260863, + 260895, + 260927, + 260959, + 260991, + 261023, + 261055, + 261087, + 261119, + 261151, + 261183, + 261215, + 261247, + 261279, + 261311, + 261343, + 261375, + 261407, + 261439, + 261471, + 261503, + 261535, + 261567, + 261599, + 261631, + 261663, + 261695, + 261727, + 261759, + 261791, + 261823, + 261855, + 261887, + 261919, + 261951, + 261983, + 262015, + 262047, + 262079, + 262111, + 262143, + 262175, + 262207, + 262239, + 262271, + 262303, + 262335, + 262367, + 262399, + 262431, + 262463, + 262495, + 262527, + 262559, + 262591, + 262623, + 262655, + 262687, + 262719, + 262751, + 262783, + 262815, + 262847, + 262879, + 262911, + 262943, + 262975, + 263007, + 263039, + 263071, + 263103, + 263135, + 263167, + 263199, + 263231, + 263263, + 263295, + 263327, + 263359, + 263391, + 263423, + 263455, + 263487, + 263519, + 263551, + 263583, + 263615, + 263647, + 263679, + 263711, + 263743, + 263775, + 263807, + 263839, + 263871, + 263903, + 263935, + 263967, + 263999, + 264031, + 264063, + 264095, + 264127, + 264159, + 264191, + 264223, + 264255, + 264287, + 264319, + 264351, + 264383, + 264415, + 264447, + 264479, + 264511, + 264543, + 264575, + 264607, + 264639, + 264671, + 264703, + 264735, + 264767, + 264799, + 264831, + 264863, + 264895, + 264927, + 264959, + 264991, + 265023, + 265055, + 265087, + 265119, + 265151, + 265183, + 265215, + 265247, + 265279, + 265311, + 265343, + 265375, + 265407, + 265439, + 265471, + 265503, + 265535, + 265567, + 265599, + 265631, + 265663, + 265695, + 265727, + 265759, + 265791, + 265823, + 265855, + 265887, + 265919, + 265951, + 265983, + 266015, + 266047, + 266079, + 266111, + 266143, + 266175, + 266207, + 266239, + 266271, + 266303, + 266335, + 266367, + 266399, + 266431, + 266463, + 266495, + 266527, + 266559, + 266591, + 266623, + 266655, + 266687, + 266719, + 266751, + 266783, + 266815, + 266847, + 266879, + 266911, + 266943, + 266975, + 267007, + 267039, + 267071, + 267103, + 267135, + 267167, + 267199, + 267231, + 267263, + 267295, + 267327, + 267359, + 267391, + 267423, + 267455, + 267487, + 267519, + 267551, + 267583, + 267615, + 267647, + 267679, + 267711, + 267743, + 267775, + 267807, + 267839, + 267871, + 267903, + 267935, + 267967, + 267999, + 268031, + 268063, + 268095, + 268127, + 268159, + 268191, + 268223, + 268255, + 268287, + 268319, + 268351, + 268383, + 268415, + 268447, + 268479, + 268511, + 268543, + 268575, + 268607, + 268639, + 268671, + 268703, + 268735, + 268767, + 268799, + 268831, + 268863, + 268895, + 268927, + 268959, + 268991, + 269023, + 269055, + 269087, + 269119, + 269151, + 269183, + 269215, + 269247, + 269279, + 269311, + 269343, + 269375, + 269407, + 269439, + 269471, + 269503, + 269535, + 269567, + 269599, + 269631, + 269663, + 269695, + 269727, + 269759, + 269791, + 269823, + 269855, + 269887, + 269919, + 269951, + 269983, + 270015, + 270047, + 270079, + 270111, + 270143, + 270175, + 270207, + 270239, + 270271, + 270303, + 270335, + 270367, + 270399, + 270431, + 270463, + 270495, + 270527, + 270559, + 270591, + 270623, + 270655, + 270687, + 270719, + 270751, + 270783, + 270815, + 270847, + 270879, + 270911, + 270943, + 270975, + 271007, + 271039, + 271071, + 271103, + 271135, + 271167, + 271199, + 271231, + 271263, + 271295, + 271327, + 271359, + 271391, + 271423, + 271455, + 271487, + 271519, + 271551, + 271583, + 271615, + 271647, + 271679, + 271711, + 271743, + 271775, + 271807, + 271839, + 271871, + 271903, + 271935, + 271967, + 271999, + 272031, + 272063, + 272095, + 272127, + 272159, + 272191, + 272223, + 272255, + 272287, + 272319, + 272351, + 272383, + 272415, + 272447, + 272479, + 272511, + 272543, + 272575, + 272607, + 272639, + 272671, + 272703, + 272735, + 272767, + 272799, + 272831, + 272863, + 272895, + 272927, + 272959, + 272991, + 273023, + 273055, + 273087, + 273119, + 273151, + 273183, + 273215, + 273247, + 273279, + 273311, + 273343, + 273375, + 273407, + 273439, + 273471, + 273503, + 273535, + 273567, + 273599, + 273631, + 273663, + 273695, + 273727, + 273759, + 273791, + 273823, + 273855, + 273887, + 273919, + 273951, + 273983, + 274015, + 274047, + 274079, + 274111, + 274143, + 274175, + 274207, + 274239, + 274271, + 274303, + 274335, + 274367, + 274399, + 274431, + 274463, + 274495, + 274527, + 274559, + 274591, + 274623, + 274655, + 274687, + 274719, + 274751, + 274783, + 274815, + 274847, + 274879, + 274911, + 274943, + 274975, + 275007, + 275039, + 275071, + 275103, + 275135, + 275167, + 275199, + 275231, + 275263, + 275295, + 275327, + 275359, + 275391, + 275423, + 275455, + 275487, + 275519, + 275551, + 275583, + 275615, + 275647, + 275679, + 275711, + 275743, + 275775, + 275807, + 275839, + 275871, + 275903, + 275935, + 275967, + 275999, + 276031, + 276063, + 276095, + 276127, + 276159, + 276191, + 276223, + 276255, + 276287, + 276319, + 276351, + 276383, + 276415, + 276447, + 276479, + 276511, + 276543, + 276575, + 276607, + 276639, + 276671, + 276703, + 276735, + 276767, + 276799, + 276831, + 276863, + 276895, + 276927, + 276959, + 276991, + 277023, + 277055, + 277087, + 277119, + 277151, + 277183, + 277215, + 277247, + 277279, + 277311, + 277343, + 277375, + 277407, + 277439, + 277471, + 277503, + 277535, + 277567, + 277599, + 277631, + 277663, + 277695, + 277727, + 277759, + 277791, + 277823, + 277855, + 277887, + 277919, + 277951, + 277983, + 278015, + 278047, + 278079, + 278111, + 278143, + 278175, + 278207, + 278239, + 278271, + 278303, + 278335, + 278367, + 278399, + 278431, + 278463, + 278495, + 278527, + 278559, + 278591, + 278623, + 278655, + 278687, + 278719, + 278751, + 278783, + 278815, + 278847, + 278879, + 278911, + 278943, + 278975, + 279007, + 279039, + 279071, + 279103, + 279135, + 279167, + 279199, + 279231, + 279263, + 279295, + 279327, + 279359, + 279391, + 279423, + 279455, + 279487, + 279519, + 279551, + 279583, + 279615, + 279647, + 279679, + 279711, + 279743, + 279775, + 279807, + 279839, + 279871, + 279903, + 279935, + 279967, + 279999, + 280031, + 280063, + 280095, + 280127, + 280159, + 280191, + 280223, + 280255, + 280287, + 280319, + 280351, + 280383, + 280415, + 280447, + 280479, + 280511, + 280543, + 280575, + 280607, + 280639, + 280671, + 280703, + 280735, + 280767, + 280799, + 280831, + 280863, + 280895, + 280927, + 280959, + 280991, + 281023, + 281055, + 281087, + 281119, + 281151, + 281183, + 281215, + 281247, + 281279, + 281311, + 281343, + 281375, + 281407, + 281439, + 281471, + 281503, + 281535, + 281567, + 281599, + 281631, + 281663, + 281695, + 281727, + 281759, + 281791, + 281823, + 281855, + 281887, + 281919, + 281951, + 281983, + 282015, + 282047, + 282079, + 282111, + 282143, + 282175, + 282207, + 282239, + 282271, + 282303, + 282335, + 282367, + 282399, + 282431, + 282463, + 282495, + 282527, + 282559, + 282591, + 282623, + 282655, + 282687, + 282719, + 282751, + 282783, + 282815, + 282847, + 282879, + 282911, + 282943, + 282975, + 283007, + 283039, + 283071, + 283103, + 283135, + 283167, + 283199, + 283231, + 283263, + 283295, + 283327, + 283359, + 283391, + 283423, + 283455, + 283487, + 283519, + 283551, + 283583, + 283615, + 283647, + 283679, + 283711, + 283743, + 283775, + 283807, + 283839, + 283871, + 283903, + 283935, + 283967, + 283999, + 284031, + 284063, + 284095, + 284127, + 284159, + 284191, + 284223, + 284255, + 284287, + 284319, + 284351, + 284383, + 284415, + 284447, + 284479, + 284511, + 284543, + 284575, + 284607, + 284639, + 284671, + 284703, + 284735, + 284767, + 284799, + 284831, + 284863, + 284895, + 284927, + 284959, + 284991, + 285023, + 285055, + 285087, + 285119, + 285151, + 285183, + 285215, + 285247, + 285279, + 285311, + 285343, + 285375, + 285407, + 285439, + 285471, + 285503, + 285535, + 285567, + 285599, + 285631, + 285663, + 285695, + 285727, + 285759, + 285791, + 285823, + 285855, + 285887, + 285919, + 285951, + 285983, + 286015, + 286047, + 286079, + 286111, + 286143, + 286175, + 286207, + 286239, + 286271, + 286303, + 286335, + 286367, + 286399, + 286431, + 286463, + 286495, + 286527, + 286559, + 286591, + 286623, + 286655, + 286687, + 286719, + 286751, + 286783, + 286815, + 286847, + 286879, + 286911, + 286943, + 286975, + 287007, + 287039, + 287071, + 287103, + 287135, + 287167, + 287199, + 287231, + 287263, + 287295, + 287327, + 287359, + 287391, + 287423, + 287455, + 287487, + 287519, + 287551, + 287583, + 287615, + 287647, + 287679, + 287711, + 287743, + 287775, + 287807, + 287839, + 287871, + 287903, + 287935, + 287967, + 287999, + 288031, + 288063, + 288095, + 288127, + 288159, + 288191, + 288223, + 288255, + 288287, + 288319, + 288351, + 288383, + 288415, + 288447, + 288479, + 288511, + 288543, + 288575, + 288607, + 288639, + 288671, + 288703, + 288735, + 288767, + 288799, + 288831, + 288863, + 288895, + 288927, + 288959, + 288991, + 289023, + 289055, + 289087, + 289119, + 289151, + 289183, + 289215, + 289247, + 289279, + 289311, + 289343, + 289375, + 289407, + 289439, + 289471, + 289503, + 289535, + 289567, + 289599, + 289631, + 289663, + 289695, + 289727, + 289759, + 289791, + 289823, + 289855, + 289887, + 289919, + 289951, + 289983, + 290015, + 290047, + 290079, + 290111, + 290143, + 290175, + 290207, + 290239, + 290271, + 290303, + 290335, + 290367, + 290399, + 290431, + 290463, + 290495, + 290527, + 290559, + 290591, + 290623, + 290655, + 290687, + 290719, + 290751, + 290783, + 290815, + 290847, + 290879, + 290911, + 290943, + 290975, + 291007, + 291039, + 291071, + 291103, + 291135, + 291167, + 291199, + 291231, + 291263, + 291295, + 291327, + 291359, + 291391, + 291423, + 291455, + 291487, + 291519, + 291551, + 291583, + 291615, + 291647, + 291679, + 291711, + 291743, + 291775, + 291807, + 291839, + 291871, + 291903, + 291935, + 291967, + 291999, + 292031, + 292063, + 292095, + 292127, + 292159, + 292191, + 292223, + 292255, + 292287, + 292319, + 292351, + 292383, + 292415, + 292447, + 292479, + 292511, + 292543, + 292575, + 292607, + 292639, + 292671, + 292703, + 292735, + 292767, + 292799, + 292831, + 292863, + 292895, + 292927, + 292959, + 292991, + 293023, + 293055, + 293087, + 293119, + 293151, + 293183, + 293215, + 293247, + 293279, + 293311, + 293343, + 293375, + 293407, + 293439, + 293471, + 293503, + 293535, + 293567, + 293599, + 293631, + 293663, + 293695, + 293727, + 293759, + 293791, + 293823, + 293855, + 293887, + 293919, + 293951, + 293983, + 294015, + 294047, + 294079, + 294111, + 294143, + 294175, + 294207, + 294239, + 294271, + 294303, + 294335, + 294367, + 294399, + 294431, + 294463, + 294495, + 294527, + 294559, + 294591, + 294623, + 294655, + 294687, + 294719, + 294751, + 294783, + 294815, + 294847, + 294879, + 294911, + 294943, + 294975, + 295007, + 295039, + 295071, + 295103, + 295135, + 295167, + 295199, + 295231, + 295263, + 295295, + 295327, + 295359, + 295391, + 295423, + 295455, + 295487, + 295519, + 295551, + 295583, + 295615, + 295647, + 295679, + 295711, + 295743, + 295775, + 295807, + 295839, + 295871, + 295903, + 295935, + 295967, + 295999, + 296031, + 296063, + 296095, + 296127, + 296159, + 296191, + 296223, + 296255, + 296287, + 296319, + 296351, + 296383, + 296415, + 296447, + 296479, + 296511, + 296543, + 296575, + 296607, + 296639, + 296671, + 296703, + 296735, + 296767, + 296799, + 296831, + 296863, + 296895, + 296927, + 296959, + 296991, + 297023, + 297055, + 297087, + 297119, + 297151, + 297183, + 297215, + 297247, + 297279, + 297311, + 297343, + 297375, + 297407, + 297439, + 297471, + 297503, + 297535, + 297567, + 297599, + 297631, + 297663, + 297695, + 297727, + 297759, + 297791, + 297823, + 297855, + 297887, + 297919, + 297951, + 297983, + 298015, + 298047, + 298079, + 298111, + 298143, + 298175, + 298207, + 298239, + 298271, + 298303, + 298335, + 298367, + 298399, + 298431, + 298463, + 298495, + 298527, + 298559, + 298591, + 298623, + 298655, + 298687, + 298719, + 298751, + 298783, + 298815, + 298847, + 298879, + 298911, + 298943, + 298975, + 299007, + 299039, + 299071, + 299103, + 299135, + 299167, + 299199, + 299231, + 299263, + 299295, + 299327, + 299359, + 299391, + 299423, + 299455, + 299487, + 299519, + 299551, + 299583, + 299615, + 299647, + 299679, + 299711, + 299743, + 299775, + 299807, + 299839, + 299871, + 299903, + 299935, + 299967, + 299999, + 300031, + 300063, + 300095, + 300127, + 300159, + 300191, + 300223, + 300255, + 300287, + 300319, + 300351, + 300383, + 300415, + 300447, + 300479, + 300511, + 300543, + 300575, + 300607, + 300639, + 300671, + 300703, + 300735, + 300767, + 300799, + 300831, + 300863, + 300895, + 300927, + 300959, + 300991, + 301023, + 301055, + 301087, + 301119, + 301151, + 301183, + 301215, + 301247, + 301279, + 301311, + 301343, + 301375, + 301407, + 301439, + 301471, + 301503, + 301535, + 301567, + 301599, + 301631, + 301663, + 301695, + 301727, + 301759, + 301791, + 301823, + 301855, + 301887, + 301919, + 301951, + 301983, + 302015, + 302047, + 302079, + 302111, + 302143, + 302175, + 302207, + 302239, + 302271, + 302303, + 302335, + 302367, + 302399, + 302431, + 302463, + 302495, + 302527, + 302559, + 302591, + 302623, + 302655, + 302687, + 302719, + 302751, + 302783, + 302815, + 302847, + 302879, + 302911, + 302943, + 302975, + 303007, + 303039, + 303071, + 303103, + 303135, + 303167, + 303199, + 303231, + 303263, + 303295, + 303327, + 303359, + 303391, + 303423, + 303455, + 303487, + 303519, + 303551, + 303583, + 303615, + 303647, + 303679, + 303711, + 303743, + 303775, + 303807, + 303839, + 303871, + 303903, + 303935, + 303967, + 303999, + 304031, + 304063, + 304095, + 304127, + 304159, + 304191, + 304223, + 304255, + 304287, + 304319, + 304351, + 304383, + 304415, + 304447, + 304479, + 304511, + 304543, + 304575, + 304607, + 304639, + 304671, + 304703, + 304735, + 304767, + 304799, + 304831, + 304863, + 304895, + 304927, + 304959, + 304991, + 305023, + 305055, + 305087, + 305119, + 305151, + 305183, + 305215, + 305247, + 305279, + 305311, + 305343, + 305375, + 305407, + 305439, + 305471, + 305503, + 305535, + 305567, + 305599, + 305631, + 305663, + 305695, + 305727, + 305759, + 305791, + 305823, + 305855, + 305887, + 305919, + 305951, + 305983, + 306015, + 306047, + 306079, + 306111, + 306143, + 306175, + 306207, + 306239, + 306271, + 306303, + 306335, + 306367, + 306399, + 306431, + 306463, + 306495, + 306527, + 306559, + 306591, + 306623, + 306655, + 306687, + 306719, + 306751, + 306783, + 306815, + 306847, + 306879, + 306911, + 306943, + 306975, + 307007, + 307039, + 307071, + 307103, + 307135, + 307167, + 307199, + 307231, + 307263, + 307295, + 307327, + 307359, + 307391, + 307423, + 307455, + 307487, + 307519, + 307551, + 307583, + 307615, + 307647, + 307679, + 307711, + 307743, + 307775, + 307807, + 307839, + 307871, + 307903, + 307935, + 307967, + 307999, + 308031, + 308063, + 308095, + 308127, + 308159, + 308191, + 308223, + 308255, + 308287, + 308319, + 308351, + 308383, + 308415, + 308447, + 308479, + 308511, + 308543, + 308575, + 308607, + 308639, + 308671, + 308703, + 308735, + 308767, + 308799, + 308831, + 308863, + 308895, + 308927, + 308959, + 308991, + 309023, + 309055, + 309087, + 309119, + 309151, + 309183, + 309215, + 309247, + 309279, + 309311, + 309343, + 309375, + 309407, + 309439, + 309471, + 309503, + 309535, + 309567, + 309599, + 309631, + 309663, + 309695, + 309727, + 309759, + 309791, + 309823, + 309855, + 309887, + 309919, + 309951, + 309983, + 310015, + 310047, + 310079, + 310111, + 310143, + 310175, + 310207, + 310239, + 310271, + 310303, + 310335, + 310367, + 310399, + 310431, + 310463, + 310495, + 310527, + 310559, + 310591, + 310623, + 310655, + 310687, + 310719, + 310751, + 310783, + 310815, + 310847, + 310879, + 310911, + 310943, + 310975, + 311007, + 311039, + 311071, + 311103, + 311135, + 311167, + 311199, + 311231, + 311263, + 311295, + 311327, + 311359, + 311391, + 311423, + 311455, + 311487, + 311519, + 311551, + 311583, + 311615, + 311647, + 311679, + 311711, + 311743, + 311775, + 311807, + 311839, + 311871, + 311903, + 311935, + 311967, + 311999, + 312031, + 312063, + 312095, + 312127, + 312159, + 312191, + 312223, + 312255, + 312287, + 312319, + 312351, + 312383, + 312415, + 312447, + 312479, + 312511, + 312543, + 312575, + 312607, + 312639, + 312671, + 312703, + 312735, + 312767, + 312799, + 312831, + 312863, + 312895, + 312927, + 312959, + 312991, + 313023, + 313055, + 313087, + 313119, + 313151, + 313183, + 313215, + 313247, + 313279, + 313311, + 313343, + 313375, + 313407, + 313439, + 313471, + 313503, + 313535, + 313567, + 313599, + 313631, + 313663, + 313695, + 313727, + 313759, + 313791, + 313823, + 313855, + 313887, + 313919, + 313951, + 313983, + 314015, + 314047, + 314079, + 314111, + 314143, + 314175, + 314207, + 314239, + 314271, + 314303, + 314335, + 314367, + 314399, + 314431, + 314463, + 314495, + 314527, + 314559, + 314591, + 314623, + 314655, + 314687, + 314719, + 314751, + 314783, + 314815, + 314847, + 314879, + 314911, + 314943, + 314975, + 315007, + 315039, + 315071, + 315103, + 315135, + 315167, + 315199, + 315231, + 315263, + 315295, + 315327, + 315359, + 315391, + 315423, + 315455, + 315487, + 315519, + 315551, + 315583, + 315615, + 315647, + 315679, + 315711, + 315775, + 315807, + 315839, + 315871, + 315903, + 315935, + 315967, + 315999, + 316031, + 316063, + 316095, + 316127, + 316159, + 316191, + 316223, + 316255, + 316287, + 316319, + 316351, + 316383, + 316415, + 316447, + 316479, + 316511, + 316543, + 316575, + 316607, + 316639, + 316671, + 316703, + 316735, + 316767, + 316799, + 316831, + 316863, + 316895, + 316927, + 316959, + 316991, + 317023, + 317055, + 317087, + 317119, + 317151, + 317183, + 317215, + 317247, + 317279, + 317311, + 317343, + 317375, + 317407, + 317439, + 317471, + 317503, + 317535, + 317567, + 317599, + 317631, + 317663, + 317695, + 317727, + 317759, + 317791, + 317823, + 317855, + 317887, + 317919, + 317951, + 317983, + 318015, + 318047, + 318079, + 318111, + 318143, + 318175, + 318207, + 318239, + 318271, + 318303, + 318335, + 318367, + 318399, + 318431, + 318463, + 318495, + 318527, + 318559, + 318591, + 318623, + 318655, + 318687, + 318719, + 318751, + 318783, + 318815, + 318847, + 318879, + 318911, + 318943, + 318975, + 319007, + 319039, + 319071, + 319103, + 319135, + 319167, + 319199, + 319231, + 319263, + 319295, + 319327, + 319359, + 319391, + 319423, + 319455, + 319487, + 319519, + 319551, + 319583, + 319615, + 319647, + 319679, + 319711, + 319743, + 319775, + 319807, + 319839, + 319871, + 319903, + 319935, + 319967, + 319999, + 320031, + 320063, + 320095, + 320127, + 320159, + 320191, + 320223, + 320255, + 320287, + 320319, + 320351, + 320383, + 320415, + 320447, + 320479, + 320511, + 320543, + 320575, + 320607, + 320639, + 320671, + 320703, + 320735, + 320767, + 320799, + 320831, + 320863, + 320895, + 320927, + 320959, + 320991, + 321023, + 321055, + 321087, + 321119, + 321151, + 321183, + 321215, + 321247, + 321279, + 321311, + 321343, + 321375, + 321407, + 321439, + 321471, + 321503, + 321535, + 321567, + 321599, + 321631, + 321663, + 321695, + 321727, + 321759, + 321791, + 321823, + 321855, + 321887, + 321919, + 321951, + 321983, + 322015, + 322047, + 322079, + 322111, + 322143, + 322175, + 322207, + 322239, + 322271, + 322303, + 322335, + 322367, + 322399, + 322431, + 322463, + 322495, + 322527, + 322559, + 322591, + 322623, + 322655, + 322687, + 322719, + 322751, + 322783, + 322815, + 322847, + 322879, + 322911, + 322943, + 322975, + 323007, + 323039, + 323071, + 323103, + 323135, + 323167, + 323199, + 323231, + 323263, + 323295, + 323327, + 323359, + 323391, + 323423, + 323455, + 323487, + 323519, + 323551, + 323583, + 323615, + 323647, + 323679, + 323711, + 323743, + 323775, + 323807, + 323839, + 323871, + 323903, + 323935, + 323967, + 323999, + 324031, + 324063, + 324095, + 324127, + 324159, + 324191, + 324223, + 324255, + 324287, + 324319, + 324351, + 324383, + 324415, + 324447, + 324479, + 324511, + 324543, + 324575, + 324607, + 324639, + 324671, + 324703, + 324735, + 324767, + 324799, + 324831, + 324863, + 324895, + 324927, + 324959, + 324991, + 325023, + 325055, + 325087, + 325119, + 325151, + 325183, + 325215, + 325247, + 325279, + 325311, + 325343, + 325375, + 325407, + 325439, + 325471, + 325503, + 325535, + 325567, + 325599, + 325631, + 325663, + 325695, + 325727, + 325759, + 325791, + 325823, + 325855, + 325887, + 325919, + 325951, + 325983, + 326015, + 326047, + 326079, + 326111, + 326143, + 326175, + 326207, + 326239, + 326271, + 326303, + 326335, + 326367, + 326399, + 326431, + 326463, + 326495, + 326527, + 326559, + 326591, + 326623, + 326655, + 326687, + 326719, + 326751, + 326783, + 326815, + 326847, + 326879, + 326911, + 326943, + 326975, + 327007, + 327039, + 327071, + 327103, + 327135, + 327167, + 327199, + 327231, + 327263, + 327295, + 327327, + 327359, + 327391, + 327423, + 327455, + 327487, + 327519, + 327551, + 327583, + 327615, + 327647, + 327679, + 327711, + 327743, + 327775, + 327807, + 327839, + 327871, + 327903, + 327935, + 327967, + 327999, + 328031, + 328063, + 328095, + 328127, + 328159, + 328191, + 328223, + 328255, + 328287, + 328319, + 328351, + 328383, + 328415, + 328447, + 328479, + 328511, + 328543, + 328575, + 328607, + 328639, + 328671, + 328703, + 328735, + 328767, + 328799, + 328831, + 328863, + 328895, + 328927, + 328959, + 328991, + 329023, + 329055, + 329087, + 329119, + 329151, + 329183, + 329215, + 329247, + 329279, + 329311, + 329343, + 329375, + 329407, + 329439, + 329471, + 329503, + 329535, + 329567, + 329599, + 329631, + 329663, + 329695, + 329727, + 329759, + 329791, + 329823, + 329855, + 329887, + 329919, + 329951, + 329983, + 330015, + 330047, + 330079, + 330111, + 330143, + 330175, + 330207, + 330239, + 330271, + 330303, + 330335, + 330367, + 330399, + 330431, + 330463, + 330495, + 330527, + 330559, + 330591, + 330623, + 330655, + 330687, + 330719, + 330751, + 330783, + 330815, + 330847, + 330879, + 330911, + 330943, + 330975, + 331007, + 331039, + 331071, + 331103, + 331135, + 331167, + 331199, + 331231, + 331263, + 331295, + 331327, + 331359, + 331391, + 331423, + 331455, + 331487, + 331519, + 331551, + 331583, + 331615, + 331647, + 331679, + 331711, + 331743, + 331775, + 331807, + 331839, + 331871, + 331903, + 331935, + 331967, + 331999, + 332031, + 332063, + 332095, + 332127, + 332159, + 332191, + 332223, + 332255, + 332287, + 332319, + 332351, + 332383, + 332415, + 332447, + 332479, + 332511, + 332543, + 332575, + 332607, + 332639, + 332671, + 332703, + 332735, + 332767, + 332799, + 332831, + 332863, + 332895, + 332927, + 332959, + 332991, + 333023, + 333055, + 333087, + 333119, + 333151, + 333183, + 333215, + 333247, + 333279, + 333311, + 333343, + 333375, + 333407, + 333439, + 333471, + 333503, + 333535, + 333567, + 333599, + 333631, + 333663, + 333695, + 333727, + 333759, + 333791, + 333823, + 333855, + 333887, + 333919, + 333951, + 333983, + 334015, + 334047, + 334079, + 334111, + 334143, + 334175, + 334207, + 334239, + 334271, + 334303, + 334335, + 334367, + 334399, + 334431, + 334463, + 334495, + 334527, + 334559, + 334591, + 334623, + 334655, + 334687, + 334719, + 334751, + 334783, + 334815, + 334847, + 334879, + 334911, + 334943, + 334975, + 335007, + 335039, + 335071, + 335103, + 335135, + 335167, + 335199, + 335231, + 335263, + 335295, + 335327, + 335359, + 335391, + 335423, + 335455, + 335487, + 335519, + 335551, + 335583, + 335615, + 335647, + 335679, + 335711, + 335743, + 335775, + 335807, + 335839, + 335871, + 335903, + 335935, + 335967, + 335999, + 336031, + 336063, + 336095, + 336127, + 336159, + 336191, + 336223, + 336255, + 336287, + 336319, + 336351, + 336383, + 336415, + 336447, + 336479, + 336511, + 336543, + 336575, + 336607, + 336639, + 336671, + 336703, + 336735, + 336767, + 336799, + 336831, + 336863, + 336895, + 336927, + 336959, + 336991, + 337023, + 337055, + 337087, + 337119, + 337151, + 337183, + 337215, + 337247, + 337279, + 337311, + 337343, + 337375, + 337407, + 337439, + 337471, + 337503, + 337535, + 337567, + 337599, + 337631, + 337663, + 337695, + 337727, + 337759, + 337791, + 337823, + 337855, + 337887, + 337919, + 337951, + 337983, + 338015, + 338047, + 338079, + 338111, + 338143, + 338175, + 338207, + 338239, + 338271, + 338303, + 338335, + 338367, + 338399, + 338431, + 338463, + 338495, + 338527, + 338559, + 338591, + 338623, + 338655, + 338687, + 338719, + 338751, + 338783, + 338815, + 338847, + 338879, + 338911, + 338943, + 338975, + 339007, + 339039, + 339071, + 339103, + 339135, + 339167, + 339199, + 339231, + 339263, + 339295, + 339327, + 339359, + 339391, + 339423, + 339455, + 339487, + 339519, + 339551, + 339583, + 339615, + 339647, + 339679, + 339711, + 339743, + 339775, + 339807, + 339839, + 339871, + 339903, + 339935, + 339967, + 339999, + 340031, + 340063, + 340095, + 340127, + 340159, + 340191, + 340223, + 340255, + 340287, + 340319, + 340351, + 340383, + 340415, + 340447, + 340479, + 340511, + 340543, + 340575, + 340607, + 340639, + 340671, + 340703, + 340735, + 340767, + 340799, + 340831, + 340863, + 340895, + 340927, + 340959, + 340991, + 341023, + 341055, + 341087, + 341119, + 341151, + 341183, + 341215, + 341247, + 341279, + 341311, + 341343, + 341375, + 341407, + 341439, + 341471, + 341503, + 341535, + 341567, + 341599, + 341631, + 341663, + 341695, + 341727, + 341759, + 341791, + 341823, + 341855, + 341887, + 341919, + 341951, + 341983, + 342015, + 342047, + 342079, + 342111, + 342143, + 342175, + 342207, + 342239, + 342271, + 342303, + 342335, + 342367, + 342399, + 342431, + 342463, + 342495, + 342527, + 342559, + 342591, + 342623, + 342655, + 342687, + 342719, + 342751, + 342783, + 342815, + 342847, + 342879, + 342911, + 342943, + 342975, + 343007, + 343039, + 343071, + 343103, + 343135, + 343167, + 343199, + 343231, + 343263, + 343295, + 343327, + 343359, + 343391, + 343423, + 343455, + 343487, + 343519, + 343551, + 343583, + 343615, + 343647, + 343679, + 343711, + 343743, + 343775, + 343807, + 343839, + 343871, + 343903, + 343935, + 343967, + 343999, + 344031, + 344063, + 344095, + 344127, + 344159, + 344191, + 344223, + 344255, + 344287, + 344319, + 344351, + 344383, + 344415, + 344447, + 344479, + 344511, + 344543, + 344575, + 344607, + 344639, + 344671, + 344703, + 344735, + 344767, + 344799, + 344831, + 344863, + 344895, + 344927, + 344959, + 344991, + 345023, + 345055, + 345087, + 345119, + 345151, + 345183, + 345215, + 345247, + 345279, + 345311, + 345343, + 345375, + 345407, + 345439, + 345471, + 345503, + 345535, + 345567, + 345599, + 345631, + 345663, + 345695, + 345727, + 345759, + 345791, + 345823, + 345855, + 345887, + 345919, + 345951, + 345983, + 346015, + 346047, + 346079, + 346111, + 346143, + 346175, + 346207, + 346239, + 346271, + 346303, + 346335, + 346367, + 346399, + 346431, + 346463, + 346495, + 346527, + 346559, + 346591, + 346623, + 346655, + 346687, + 346719, + 346751, + 346783, + 346815, + 346847, + 346879, + 346911, + 346943, + 346975, + 347007, + 347039, + 347071, + 347103, + 347135, + 347167, + 347199, + 347231, + 347263, + 347295, + 347327, + 347359, + 347391, + 347423, + 347455, + 347487, + 347519, + 347551, + 347583, + 347615, + 347647, + 347679, + 347711, + 347743, + 347775, + 347807, + 347839, + 347871, + 347903, + 347935, + 347967, + 347999, + 348031, + 348063, + 348095, + 348127, + 348159, + 348191, + 348223, + 348255, + 348287, + 348319, + 348383, + 348415, + 348447, + 348479, + 348511, + 348543, + 348575, + 348607, + 348639, + 348671, + 348703, + 348735, + 348767, + 348799, + 348831, + 348863, + 348895, + 348927, + 348959, + 348991, + 349023, + 349055, + 349087, + 349119, + 349151, + 349183, + 349215, + 349247, + 349279, + 349311, + 349343, + 349375, + 349407, + 349439, + 349471, + 349503, + 349535, + 349567, + 349599, + 349631, + 349663, + 349695, + 349727, + 349759, + 349791, + 349823, + 349855, + 349887, + 349919, + 349951, + 349983, + 350015, + 350047, + 350079, + 350111, + 350143, + 350175, + 350207, + 350239, + 350271, + 350303, + 350335, + 350367, + 350399, + 350431, + 350463, + 350495, + 350527, + 350559, + 350591, + 350623, + 350655, + 350687, + 350719, + 350751, + 350783, + 350815, + 350847, + 350879, + 350911, + 350943, + 350975, + 351007, + 351039, + 351071, + 351103, + 351135, + 351167, + 351199, + 351231, + 351263, + 351295, + 351327, + 351359, + 351391, + 351423, + 351455, + 351487, + 351519, + 351551, + 351583, + 351615, + 351647, + 351679, + 351711, + 351743, + 351775, + 351807, + 351839, + 351871, + 351903, + 351935, + 351967, + 351999, + 352031, + 352063, + 352095, + 352127, + 352159, + 352191, + 352223, + 352255, + 352287, + 352319, + 352351, + 352383, + 352415, + 352447, + 352479, + 352511, + 352543, + 352575, + 352607, + 352639, + 352671, + 352703, + 352735, + 352767, + 352799, + 352831, + 352863, + 352895, + 352927, + 352959, + 352991, + 353023, + 353055, + 353087, + 353119, + 353151, + 353183, + 353215, + 353247, + 353279, + 353311, + 353343, + 353375, + 353407, + 353439, + 353471, + 353503, + 353535, + 353567, + 353599, + 353631, + 353663, + 353695, + 353727, + 353759, + 353791, + 353823, + 353855, + 353887, + 353919, + 353951, + 353983, + 354015, + 354047, + 354079, + 354111, + 354143, + 354175, + 354207, + 354239, + 354271, + 354303, + 354335, + 354367, + 354399, + 354431, + 354463, + 354495, + 354527, + 354559, + 354591, + 354623, + 354655, + 354687, + 354719, + 354751, + 354783, + 354815, + 354847, + 354879, + 354911, + 354943, + 354975, + 355007, + 355039, + 355071, + 355103, + 355135, + 355167, + 355199, + 355231, + 355263, + 355295, + 355327, + 355359, + 355391, + 355423, + 355455, + 355487, + 355519, + 355551, + 355583, + 355615, + 355647, + 355679, + 355711, + 355743, + 355775, + 355807, + 355839, + 355871, + 355903, + 355935, + 355967, + 355999, + 356031, + 356063, + 356095, + 356127, + 356159, + 356191, + 356223, + 356255, + 356319, + 356351, + 356383, + 356447, + 356479, + 356511, + 356543, + 356575, + 356607, + 356639, + 356671, + 356703, + 356735, + 356767, + 356799, + 356831, + 356863, + 356895, + 356927, + 356959, + 356991, + 357023, + 357055, + 357087, + 357119, + 357151, + 357183, + 357215, + 357247, + 357279, + 357311, + 357343, + 357375, + 357407, + 357439, + 357471, + 357503, + 357535, + 357567, + 357599, + 357631, + 357663, + 357695, + 357727, + 357759, + 357791, + 357823, + 357855, + 357887, + 357919, + 357951, + 357983, + 358015, + 358047, + 358079, + 358111, + 358143, + 358175, + 358207, + 358239, + 358271, + 358303, + 358335, + 358367, + 358399, + 358431, + 358463, + 358495, + 358527, + 358559, + 358623, + 358655, + 358687, + 358719, + 358751, + 358783, + 358815, + 358847, + 358879, + 358911, + 358943, + 358975, + 359007, + 359039, + 359071, + 359103, + 359135, + 359167, + 359199, + 359231, + 359295, + 359327, + 359359, + 359391, + 359423, + 359455, + 359487, + 359519, + 359551, + 359583, + 359615, + 359647, + 359679, + 359711, + 359743, + 359775, + 359807, + 359839, + 359871, + 359903, + 359935, + 359967, + 359999, + 360031, + 360063, + 360095, + 360127, + 360159, + 360191, + 360223, + 360255, + 360287, + 360319, + 360351, + 360383, + 360415, + 360447, + 360479, + 360511, + 360543, + 360575, + 360607, + 360639, + 360671, + 360703, + 360735, + 360767, + 360799, + 360831, + 360863, + 360895, + 360927, + 360959, + 360991, + 361023, + 361055, + 361087, + 361119, + 361151, + 361183, + 361215, + 361247, + 361279, + 361311, + 361343, + 361375, + 361407, + 361439, + 361471, + 361503, + 361535, + 361567, + 361599, + 361631, + 361663, + 361695, + 361727, + 361759, + 361791, + 361823, + 361855, + 361887, + 361919, + 361951, + 361983, + 362015, + 362047, + 362079, + 362111, + 362143, + 362175, + 362207, + 362239, + 362271, + 362303, + 362335, + 362367, + 362399, + 362431, + 362463, + 362495, + 362527, + 362559, + 362591, + 362623, + 362655, + 362687, + 362719, + 362751, + 362783, + 362815, + 362847, + 362879, + 362911, + 362943, + 362975, + 363007, + 363039, + 363071, + 363103, + 363135, + 363167, + 363199, + 363231, + 363263, + 363295, + 363327, + 363359, + 363391, + 363423, + 363455, + 363487, + 363519, + 363551, + 363583, + 363615, + 363647, + 363679, + 363711, + 363743, + 363775, + 363807, + 363839, + 363871, + 363903, + 363935, + 363967, + 363999, + 364031, + 364063, + 364095, + 364127, + 364159, + 364191, + 364223, + 364255, + 364287, + 364319, + 364351, + 364383, + 364415, + 364447, + 364479, + 364511, + 364543, + 364575, + 364607, + 364639, + 364671, + 364703, + 364735, + 364767, + 364799, + 364831, + 364863, + 364895, + 364927, + 364959, + 364991, + 365023, + 365055, + 365087, + 365119, + 365151, + 365183, + 365215, + 365247, + 365279, + 365311, + 365343, + 365375, + 365407, + 365439, + 365471, + 365503, + 365535, + 365567, + 365599, + 365631, + 365663, + 365695, + 365727, + 365759, + 365791, + 365823, + 365855, + 365887, + 365919, + 365951, + 365983, + 366015, + 366047, + 366079, + 366111, + 366143, + 366175, + 366207, + 366239, + 366271, + 366303, + 366335, + 366367, + 366399, + 366431, + 366463, + 366495, + 366527, + 366559, + 366591, + 366623, + 366655, + 366687, + 366719, + 366751, + 366783, + 366815, + 366847, + 366879, + 366911, + 366943, + 366975, + 367007, + 367039, + 367103, + 367135, + 367167, + 367199, + 367231, + 367263, + 367295, + 367327, + 367359, + 367391, + 367423, + 367455, + 367487, + 367519, + 367551, + 367583, + 367615, + 367647, + 367679, + 367711, + 367743, + 367775, + 367807, + 367839, + 367871, + 367903, + 367935, + 367967, + 367999, + 368031, + 368063, + 368095, + 368127, + 368159, + 368191, + 368223, + 368255, + 368287, + 368319, + 368383, + 368415, + 368447, + 368479, + 368511, + 368543, + 368575, + 368607, + 368639, + 368703, + 368735, + 368767, + 368799, + 368831, + 368863, + 368895, + 368927, + 368959, + 368991, + 369023, + 369055, + 369087, + 369119, + 369151, + 369183, + 369215, + 369247, + 369279, + 369311, + 369343, + 369375, + 369407, + 369439, + 369471, + 369503, + 369535, + 369567, + 369599, + 369631, + 369663, + 369695, + 369727, + 369759, + 369791, + 369823, + 369855, + 369887, + 369919, + 369951, + 369983, + 370015, + 370047, + 370079, + 370111, + 370143, + 370175, + 370207, + 370239, + 370271, + 370303, + 370335, + 370367, + 370399, + 370431, + 370463, + 370495, + 370527, + 370559, + 370591, + 370623, + 370655, + 370687, + 370719, + 370751, + 370783, + 370815, + 370847, + 370879, + 370911, + 370943, + 370975, + 371007, + 371039, + 371071, + 371103, + 371135, + 371167, + 371199, + 371231, + 371263, + 371295, + 371327, + 371359, + 371391, + 371423, + 371455, + 371487, + 371519, + 371551, + 371583, + 371615, + 371647, + 371679, + 371711, + 371743, + 371775, + 371807, + 371839, + 371871, + 371903, + 371935, + 371967, + 371999, + 372031, + 372063, + 372095, + 372127, + 372159, + 372191, + 372223, + 372255, + 372287, + 372319, + 372351, + 372383, + 372415, + 372447, + 372479, + 372511, + 372543, + 372575, + 372607, + 372639, + 372671, + 372703, + 372735, + 372767, + 372799, + 372831, + 372863, + 372895, + 372927, + 372959, + 372991, + 373023, + 373055, + 373087, + 373119, + 373151, + 373183, + 373215, + 373247, + 373279, + 373311, + 373343, + 373375, + 373407, + 373439, + 373471, + 373503, + 373535, + 373567, + 373599, + 373631, + 373663, + 373695, + 373727, + 373759, + 373791, + 373823, + 373855, + 373887, + 373919, + 373951, + 373983, + 374015, + 374047, + 374079, + 374111, + 374143, + 374175, + 374207, + 374239, + 374271, + 374303, + 374335, + 374367, + 374399, + 374431, + 374463, + 374495, + 374527, + 374559, + 374591, + 374623, + 374687, + 374719, + 374751, + 374783, + 374815, + 374847, + 374879, + 374911, + 374943, + 374975, + 375007, + 375039, + 375071, + 375103, + 375135, + 375167, + 375199, + 375231, + 375263, + 375295, + 375327, + 375359, + 375391, + 375423, + 375455, + 375487, + 375519, + 375551, + 375583, + 375615, + 375647, + 375679, + 375711, + 375743, + 375775, + 375807, + 375839, + 375871, + 375935, + 375967, + 375999, + 376031, + 376063, + 376095, + 376127, + 376159, + 376191, + 376223, + 376255, + 376287, + 376319, + 376351, + 376383, + 376415, + 376447, + 376479, + 376511, + 376543, + 376575, + 376607, + 376639, + 376671, + 376703, + 376735, + 376767, + 376799, + 376831, + 376863, + 376895, + 376927, + 376959, + 376991, + 377023, + 377055, + 377087, + 377119, + 377151, + 377183, + 377215, + 377247, + 377279, + 377311, + 377343, + 377375, + 377407, + 377439, + 377471, + 377503, + 377535, + 377567, + 377599, + 377631, + 377663, + 377695, + 377727, + 377759, + 377791, + 377823, + 377855, + 377887, + 377919, + 377951, + 377983, + 378015, + 378047, + 378079, + 378111, + 378143, + 378175, + 378207, + 378239, + 378271, + 378303, + 378335, + 378367, + 378399, + 378431, + 378463, + 378495, + 378527, + 378559, + 378591, + 378623, + 378655, + 378687, + 378719, + 378751, + 378783, + 378815, + 378847, + 378879, + 378911, + 378943, + 378975, + 379007, + 379039, + 379071, + 379103, + 379135, + 379167, + 379199, + 379231, + 379263, + 379295, + 379327, + 379359, + 379391, + 379423, + 379455, + 379487, + 379519, + 379551, + 379583, + 379615, + 379647, + 379679, + 379711, + 379743, + 379775, + 379807, + 379839, + 379871, + 379903, + 379967, + 379999, + 380031, + 380063, + 380095, + 380127, + 380159, + 380191, + 380223, + 380255, + 380287, + 380319, + 380351, + 380383, + 380415, + 380447, + 380479, + 380511, + 380543, + 380575, + 380607, + 380639, + 380671, + 380703, + 380735, + 380767, + 380799, + 380831, + 380863, + 380895, + 380927, + 380959, + 380991, + 381023, + 381055, + 381087, + 381119, + 381151, + 381183, + 381215, + 381247, + 381279, + 381311, + 381343, + 381375, + 381407, + 381471, + 381503, + 381535, + 381567, + 381599, + 381631, + 381663, + 381695, + 381727, + 381759, + 381791, + 381823, + 381855, + 381887, + 381919, + 381951, + 381983, + 382015, + 382047, + 382079, + 382111, + 382143, + 382175, + 382207, + 382239, + 382271, + 382303, + 382335, + 382367, + 382399, + 382431, + 382463, + 382495, + 382527, + 382559, + 382591, + 382623, + 382655, + 382687, + 382719, + 382751, + 382783, + 382815, + 382847, + 382879, + 382911, + 382943, + 382975, + 383007, + 383071, + 383103, + 383135, + 383167, + 383199, + 383231, + 383263, + 383295, + 383327, + 383359, + 383391, + 383423, + 383455, + 383487, + 383519, + 383551, + 383583, + 383615, + 383647, + 383679, + 383743, + 383775, + 383807, + 383839, + 383871, + 383903, + 383935, + 383967, + 384031, + 384063, + 384127, + 384159, + 384191, + 384223, + 384255, + 384287, + 384319, + 384351, + 384383, + 384415, + 384447, + 384479, + 384511, + 384543, + 384575, + 384607, + 384639, + 384671, + 384703, + 384735, + 384767, + 384799, + 384831, + 384863, + 384895, + 384927, + 384959, + 384991, + 385023, + 385055, + 385087, + 385119, + 385151, + 385183, + 385215, + 385247, + 385279, + 385311, + 385343, + 385375, + 385407, + 385439, + 385471, + 385503, + 385535, + 385567, + 385599, + 385631, + 385663, + 385695, + 385727, + 385759, + 385791, + 385823, + 385855, + 385887, + 385919, + 385951, + 385983, + 386015, + 386047, + 386079, + 386111, + 386143, + 386175, + 386207, + 386239, + 386271, + 386303, + 386335, + 386367, + 386399, + 386431, + 386463, + 386495, + 386527, + 386559, + 386591, + 386623, + 386687, + 386719, + 386751, + 386783, + 386815, + 386847, + 386879, + 386911, + 386943, + 386975, + 387007, + 387039, + 387071, + 387103, + 387135, + 387167, + 387199, + 387231, + 387263, + 387295, + 387327, + 387359, + 387391, + 387423, + 387455, + 387487, + 387519, + 387551, + 387583, + 387615, + 387647, + 387679, + 387711, + 387743, + 387775, + 387807, + 387839, + 387871, + 387903, + 387935, + 387967, + 387999, + 388031, + 388063, + 388127, + 388159, + 388191, + 388223, + 388255, + 388287, + 388319, + 388351, + 388383, + 388415, + 388447, + 388479, + 388511, + 388543, + 388575, + 388607, + 388639, + 388671, + 388703, + 388735, + 388767, + 388799, + 388831, + 388863, + 388895, + 388927, + 388959, + 388991, + 389023, + 389055, + 389087, + 389119, + 389183, + 389215, + 389247, + 389279, + 389311, + 389343, + 389375, + 389407, + 389439, + 389471, + 389503, + 389535, + 389567, + 389599, + 389631, + 389663, + 389695, + 389727, + 389759, + 389791, + 389823, + 389855, + 389887, + 389919, + 389951, + 389983, + 390015, + 390047, + 390079, + 390111, + 390143, + 390175, + 390207, + 390239, + 390271, + 390303, + 390335, + 390367, + 390399, + 390431, + 390463, + 390495, + 390527, + 390559, + 390591, + 390623, + 390655, + 390687, + 390719, + 390751, + 390783, + 390815, + 390847, + 390879, + 390911, + 390943, + 390975, + 391007, + 391039, + 391071, + 391103, + 391135, + 391167, + 391199, + 391231, + 391263, + 391295, + 391327, + 391359, + 391391, + 391423, + 391455, + 391487, + 391519, + 391551, + 391583, + 391615, + 391647, + 391679, + 391711, + 391743, + 391775, + 391807, + 391839, + 391871, + 391903, + 391935, + 391967, + 391999, + 392031, + 392063, + 392095, + 392127, + 392159, + 392191, + 392223, + 392255, + 392287, + 392319, + 392351, + 392383, + 392415, + 392447, + 392479, + 392511, + 392543, + 392575, + 392607, + 392639, + 392671, + 392703, + 392735, + 392767, + 392799, + 392831, + 392863, + 392895, + 392927, + 392959, + 392991, + 393023, + 393055, + 393087, + 393119, + 393151, + 393183, + 393215, + 393247, + 393279, + 393311, + 393343, + 393375, + 393407, + 393439, + 393471, + 393503, + 393535, + 393567, + 393599, + 393631, + 393663, + 393695, + 393727, + 393759, + 393791, + 393823, + 393855, + 393887, + 393919, + 393951, + 393983, + 394015, + 394047, + 394079, + 394111, + 394143, + 394175, + 394207, + 394239, + 394271, + 394303, + 394335, + 394367, + 394431, + 394463, + 394495, + 394527, + 394559, + 394591, + 394623, + 394655, + 394687, + 394719, + 394751, + 394783, + 394815, + 394847, + 394879, + 394911, + 394943, + 394975, + 395007, + 395039, + 395071, + 395103, + 395135, + 395167, + 395199, + 395231, + 395263, + 395295, + 395327, + 395359, + 395391, + 395423, + 395455, + 395487, + 395519, + 395551, + 395583, + 395615, + 395647, + 395679, + 395711, + 395743, + 395775, + 395807, + 395839, + 395871, + 395903, + 395935, + 395967, + 396031, + 396063, + 396095, + 396127, + 396159, + 396191, + 396223, + 396255, + 396287, + 396319, + 396351, + 396383, + 396415, + 396447, + 396511, + 396543, + 396575, + 396607, + 396639, + 396671, + 396703, + 396735, + 396767, + 396799, + 396831, + 396863, + 396895, + 396927, + 396959, + 396991, + 397023, + 397055, + 397087, + 397119, + 397151, + 397183, + 397215, + 397247, + 397279, + 397311, + 397343, + 397375, + 397407, + 397439, + 397471, + 397503, + 397535, + 397567, + 397599, + 397631, + 397663, + 397695, + 397727, + 397759, + 397791, + 397823, + 397855, + 397887, + 397919, + 397951, + 397983, + 398015, + 398047, + 398079, + 398111, + 398143, + 398175, + 398207, + 398239, + 398271, + 398303, + 398335, + 398367, + 398399, + 398431, + 398463, + 398495, + 398527, + 398559, + 398591, + 398623, + 398655, + 398687, + 398719, + 398751, + 398783, + 398815, + 398847, + 398879, + 398911, + 398943, + 398975, + 399007, + 399039, + 399071, + 399103, + 399135, + 399167, + 399199, + 399231, + 399295, + 399327, + 399359, + 399391, + 399423, + 399455, + 399487, + 399519, + 399551, + 399583, + 399615, + 399647, + 399679, + 399711, + 399743, + 399775, + 399807, + 399839, + 399871, + 399903, + 399935, + 399967, + 399999, + 400031, + 400063, + 400095, + 400127, + 400159, + 400191, + 400223, + 400255, + 400287, + 400351, + 400383, + 400415, + 400447, + 400479, + 400511, + 400543, + 400575, + 400607, + 400639, + 400671, + 400703, + 400735, + 400767, + 400799, + 400831, + 400863, + 400895, + 400927, + 400959, + 400991, + 401023, + 401055, + 401087, + 401119, + 401151, + 401183, + 401215, + 401247, + 401279, + 401311, + 401375, + 401407, + 401439, + 401471, + 401503, + 401535, + 401567, + 401599, + 401631, + 401663, + 401695, + 401727, + 401759, + 401791, + 401823, + 401855, + 401887, + 401919, + 401951, + 401983, + 402015, + 402047, + 402079, + 402111, + 402143, + 402175, + 402207, + 402239, + 402271, + 402303, + 402335, + 402367, + 402399, + 402431, + 402463, + 402495, + 402527, + 402559, + 402591, + 402655, + 402687, + 402719, + 402751, + 402783, + 402815, + 402847, + 402879, + 402911, + 402943, + 402975, + 403007, + 403039, + 403071, + 403103, + 403135, + 403167, + 403199, + 403263, + 403295, + 403327, + 403359, + 403391, + 403423, + 403455, + 403487, + 403519, + 403551, + 403583, + 403615, + 403647, + 403679, + 403711, + 403743, + 403775, + 403807, + 403839, + 403871, + 403903, + 403935, + 403967, + 403999, + 404031, + 404063, + 404095, + 404127, + 404159, + 404191, + 404223, + 404255, + 404287, + 404319, + 404351, + 404383, + 404415, + 404447, + 404479, + 404511, + 404543, + 404575, + 404607, + 404639, + 404671, + 404703, + 404735, + 404767, + 404799, + 404831, + 404863, + 404895, + 404927, + 404959, + 404991, + 405023, + 405055, + 405087, + 405119, + 405151, + 405183, + 405215, + 405247, + 405279, + 405311, + 405343, + 405375, + 405407, + 405439, + 405471, + 405503, + 405535, + 405567, + 405599, + 405631, + 405663, + 405695, + 405727, + 405759, + 405791, + 405823, + 405855, + 405887, + 405919, + 405951, + 405983, + 406015, + 406047, + 406079, + 406111, + 406143, + 406175, + 406207, + 406239, + 406271, + 406303, + 406335, + 406367, + 406431, + 406463, + 406495, + 406527, + 406559, + 406591, + 406623, + 406655, + 406687, + 406719, + 406751, + 406783, + 406815, + 406847, + 406879, + 406911, + 406943, + 406975, + 407007, + 407039, + 407071, + 407103, + 407135, + 407167, + 407199, + 407231, + 407263, + 407295, + 407327, + 407359, + 407391, + 407423, + 407455, + 407487, + 407519, + 407551, + 407583, + 407615, + 407647, + 407679, + 407711, + 407743, + 407775, + 407807, + 407839, + 407871, + 407903, + 407935, + 407967, + 407999, + 408031, + 408063, + 408095, + 408127, + 408159, + 408191, + 408223, + 408255, + 408287, + 408319, + 408351, + 408383, + 408415, + 408447, + 408479, + 408543, + 408575, + 408607, + 408639, + 408671, + 408703, + 408735, + 408767, + 408799, + 408831, + 408863, + 408895, + 408927, + 408959, + 408991, + 409023, + 409055, + 409087, + 409119, + 409151, + 409183, + 409215, + 409247, + 409279, + 409311, + 409343, + 409375, + 409407, + 409439, + 409471, + 409503, + 409535, + 409567, + 409599, + 409631, + 409663, + 409695, + 409727, + 409759, + 409791, + 409823, + 409855, + 409887, + 409919, + 409951, + 409983, + 410015, + 410047, + 410079, + 410111, + 410143, + 410175, + 410207, + 410239, + 410271, + 410303, + 410335, + 410367, + 410399, + 410431, + 410463, + 410495, + 410527, + 410559, + 410591, + 410623, + 410655, + 410687, + 410719, + 410751, + 410783, + 410815, + 410847, + 410879, + 410911, + 410943, + 410975, + 411007, + 411039, + 411071, + 411103, + 411135, + 411167, + 411199, + 411231, + 411263, + 411295, + 411327, + 411359, + 411391, + 411423, + 411455, + 411487, + 411519, + 411551, + 411583, + 411615, + 411647, + 411679, + 411711, + 411743, + 411775, + 411807, + 411839, + 411871, + 411903, + 411935, + 411999, + 412031, + 412063, + 412095, + 412127, + 412159, + 412191, + 412223, + 412255, + 412287, + 412319, + 412351, + 412383, + 412415, + 412447, + 412479, + 412511, + 412543, + 412575, + 412607, + 412639, + 412671, + 412703, + 412735, + 412767, + 412799, + 412831, + 412863, + 412895, + 412927, + 412959, + 412991, + 413023, + 413055, + 413087, + 413119, + 413151, + 413183, + 413247, + 413279, + 413311, + 413343, + 413375, + 413407, + 413439, + 413471, + 413503, + 413535, + 413599, + 413631, + 413663, + 413695, + 413727, + 413759, + 413791, + 413823, + 413855, + 413887, + 413919, + 413951, + 413983, + 414015, + 414047, + 414079, + 414111, + 414143, + 414175, + 414207, + 414239, + 414271, + 414303, + 414335, + 414367, + 414399, + 414431, + 414463, + 414495, + 414527, + 414559, + 414591, + 414623, + 414655, + 414687, + 414719, + 414751, + 414783, + 414815, + 414847, + 414879, + 414911, + 414943, + 414975, + 415007, + 415039, + 415071, + 415103, + 415135, + 415167, + 415199, + 415231, + 415263, + 415295, + 415327, + 415359, + 415391, + 415423, + 415455, + 415487, + 415519, + 415551, + 415583, + 415615, + 415647, + 415679, + 415711, + 415743, + 415775, + 415807, + 415839, + 415871, + 415903, + 415935, + 415967, + 415999, + 416031, + 416063, + 416095, + 416127, + 416159, + 416191, + 416223, + 416255, + 416287, + 416319, + 416351, + 416383, + 416415, + 416447, + 416479, + 416543, + 416575, + 416607, + 416639, + 416671, + 416703, + 416735, + 416767, + 416799, + 416831, + 416863, + 416895, + 416927, + 416959, + 416991, + 417023, + 417055, + 417087, + 417119, + 417151, + 417183, + 417215, + 417247, + 417279, + 417343, + 417375, + 417407, + 417439, + 417471, + 417503, + 417535, + 417567, + 417599, + 417631, + 417663, + 417695, + 417727, + 417759, + 417791, + 417823, + 417855, + 417887, + 417919, + 417951, + 417983, + 418015, + 418047, + 418079, + 418111, + 418143, + 418175, + 418207, + 418239, + 418271, + 418303, + 418335, + 418367, + 418399, + 418431, + 418463, + 418495, + 418527, + 418559, + 418591, + 418623, + 418655, + 418687, + 418719, + 418751, + 418783, + 418815, + 418847, + 418879, + 418911, + 418943, + 418975, + 419007, + 419039, + 419071, + 419103, + 419135, + 419167, + 419199, + 419231, + 419263, + 419295, + 419327, + 419359, + 419391, + 419423, + 419455, + 419487, + 419519, + 419551, + 419583, + 419615, + 419647, + 419679, + 419711, + 419743, + 419775, + 419807, + 419839, + 419871, + 419903, + 419935, + 419967, + 419999, + 420031, + 420063, + 420095, + 420127, + 420159, + 420191, + 420223, + 420255, + 420287, + 420319, + 420351, + 420383, + 420415, + 420447, + 420479, + 420511, + 420543, + 420575, + 420607, + 420639, + 420671, + 420703, + 420735, + 420767, + 420799, + 420831, + 420863, + 420895, + 420927, + 420959, + 420991, + 421023, + 421055, + 421087, + 421119, + 421151, + 421183, + 421215, + 421247, + 421279, + 421311, + 421343, + 421375, + 421407, + 421439, + 421471, + 421503, + 421535, + 421567, + 421599, + 421631, + 421663, + 421695, + 421727, + 421759, + 421791, + 421823, + 421855, + 421887, + 421919, + 421951, + 421983, + 422015, + 422047, + 422079, + 422111, + 422143, + 422175, + 422207, + 422239, + 422271, + 422303, + 422335, + 422367, + 422399, + 422431, + 422463, + 422495, + 422527, + 422559, + 422591, + 422623, + 422655, + 422687, + 422719, + 422751, + 422783, + 422815, + 422847, + 422879, + 422911, + 422943, + 422975, + 423007, + 423039, + 423071, + 423103, + 423135, + 423167, + 423199, + 423231, + 423263, + 423295, + 423359, + 423391, + 423423, + 423455, + 423487, + 423519, + 423551, + 423583, + 423615, + 423647, + 423679, + 423743, + 423775, + 423807, + 423839, + 423871, + 423935, + 423967, + 423999, + 424031, + 424063, + 424095, + 424127, + 424159, + 424191, + 424223, + 424255, + 424287, + 424319, + 424351, + 424383, + 424415, + 424447, + 424479, + 424511, + 424543, + 424575, + 424607, + 424639, + 424671, + 424703, + 424735, + 424767, + 424799, + 424831, + 424863, + 424895, + 424927, + 424959, + 424991, + 425023, + 425055, + 425087, + 425119, + 425151, + 425183, + 425215, + 425247, + 425279, + 425311, + 425343, + 425375, + 425407, + 425439, + 425471, + 425503, + 425535, + 425567, + 425599, + 425631, + 425663, + 425695, + 425727, + 425759, + 425791, + 425823, + 425855, + 425887, + 425919, + 425951, + 425983, + 426015, + 426047, + 426079, + 426111, + 426143, + 426175, + 426207, + 426239, + 426271, + 426303, + 426335, + 426367, + 426399, + 426431, + 426463, + 426495, + 426527, + 426559, + 426591, + 426623, + 426655, + 426687, + 426751, + 426783, + 426815, + 426847, + 426879, + 426911, + 426943, + 426975, + 427007, + 427039, + 427071, + 427103, + 427135, + 427167, + 427199, + 427231, + 427263, + 427295, + 427359, + 427391, + 427423, + 427455, + 427487, + 427519, + 427551, + 427583, + 427615, + 427647, + 427679, + 427711, + 427743, + 427775, + 427807, + 427839, + 427871, + 427903, + 427935, + 427967, + 427999, + 428031, + 428063, + 428095, + 428127, + 428159, + 428191, + 428223, + 428255, + 428287, + 428319, + 428351, + 428383, + 428415, + 428447, + 428479, + 428511, + 428543, + 428575, + 428607, + 428639, + 428671, + 428703, + 428735, + 428767, + 428799, + 428831, + 428863, + 428895, + 428927, + 428959, + 428991, + 429023, + 429055, + 429119, + 429151, + 429183, + 429215, + 429247, + 429311, + 429343, + 429375, + 429407, + 429439, + 429471, + 429503, + 429535, + 429567, + 429599, + 429631, + 429663, + 429727, + 429759, + 429791, + 429823, + 429855, + 429887, + 429919, + 429983, + 430015, + 430047, + 430079, + 430111, + 430143, + 430175, + 430207, + 430271, + 430303, + 430335, + 430367, + 430399, + 430431, + 430463, + 430495, + 430559, + 430591, + 430623, + 430655, + 430687, + 430719, + 430751, + 430783, + 430815, + 430879, + 430911, + 430943, + 430975, + 431007, + 431039, + 431071, + 431103, + 431135, + 431167, + 431199, + 431231, + 431263, + 431295, + 431327, + 431359, + 431391, + 431455, + 431487, + 431519, + 431551, + 431583, + 431615, + 431647, + 431679, + 431711, + 431743, + 431775, + 431807, + 431839, + 431871, + 431903, + 431935, + 431967, + 432031, + 432063, + 432095, + 432127, + 432159, + 432191, + 432223, + 432255, + 432287, + 432319, + 432351, + 432383, + 432415, + 432447, + 432479, + 432511, + 432543, + 432575, + 432607, + 432639, + 432671, + 432703, + 432735, + 432767, + 432799, + 432831, + 432863, + 432895, + 432959, + 432991, + 433023, + 433055, + 433087, + 433119, + 433151, + 433183, + 433247, + 433279, + 433311, + 433343, + 433375, + 433407, + 433439, + 433471, + 433503, + 433535, + 433567, + 433599, + 433631, + 433663, + 433695, + 433727, + 433759, + 433791, + 433823, + 433855, + 433887, + 433919, + 433951, + 433983, + 434015, + 434047, + 434079, + 434111, + 434143, + 434175, + 434207, + 434271, + 434303, + 434335, + 434367, + 434399, + 434431, + 434463, + 434495, + 434527, + 434559, + 434591, + 434623, + 434655, + 434687, + 434719, + 434751, + 434783, + 434815, + 434847, + 434879, + 434911, + 434943, + 434975, + 435007, + 435039, + 435071, + 435103, + 435135, + 435167, + 435199, + 435231, + 435263, + 435295, + 435327, + 435359, + 435391, + 435423, + 435455, + 435487, + 435519, + 435551, + 435583, + 435615, + 435647, + 435679, + 435711, + 435743, + 435775, + 435807, + 435839, + 435871, + 435903, + 435935, + 435967, + 435999, + 436031, + 436063, + 436095, + 436127, + 436159, + 436191, + 436223, + 436255, + 436287, + 436319, + 436351, + 436415, + 436447, + 436479, + 436511, + 436543, + 436575, + 436607, + 436639, + 436671, + 436703, + 436735, + 436767, + 436831, + 436863, + 436927, + 436959, + 436991, + 437023, + 437055, + 437087, + 437119, + 437151, + 437183, + 437215, + 437247, + 437279, + 437311, + 437343, + 437375, + 437407, + 437439, + 437471, + 437503, + 437535, + 437567, + 437631, + 437663, + 437695, + 437727, + 437759, + 437791, + 437823, + 437855, + 437887, + 437919, + 437951, + 437983, + 438015, + 438047, + 438079, + 438111, + 438143, + 438175, + 438207, + 438239, + 438271, + 438303, + 438335, + 438367, + 438399, + 438431, + 438463, + 438495, + 438527, + 438559, + 438591, + 438623, + 438655, + 438687, + 438719, + 438751, + 438783, + 438815, + 438847, + 438879, + 438943, + 438975, + 439007, + 439039, + 439071, + 439103, + 439135, + 439167, + 439199, + 439231, + 439263, + 439295, + 439327, + 439391, + 439423, + 439455, + 439519, + 439551, + 439583, + 439615, + 439647, + 439679, + 439711, + 439743, + 439775, + 439807, + 439839, + 439871, + 439903, + 439935, + 439967, + 439999, + 440031, + 440063, + 440095, + 440127, + 440159, + 440191, + 440223, + 440255, + 440287, + 440319, + 440351, + 440383, + 440415, + 440447, + 440479, + 440511, + 440543, + 440575, + 440607, + 440639, + 440671, + 440703, + 440735, + 440767, + 440799, + 440831, + 440863, + 440895, + 440959, + 440991, + 441023, + 441055, + 441087, + 441119, + 441151, + 441183, + 441215, + 441247, + 441279, + 441311, + 441343, + 441375, + 441407, + 441439, + 441471, + 441535, + 441567, + 441599, + 441631, + 441663, + 441695, + 441727, + 441759, + 441791, + 441823, + 441855, + 441887, + 441919, + 441951, + 441983, + 442015, + 442047, + 442079, + 442111, + 442143, + 442207, + 442239, + 442271, + 442303, + 442335, + 442367, + 442399, + 442463, + 442495, + 442527, + 442559, + 442591, + 442623, + 442655, + 442687, + 442719, + 442751, + 442783, + 442815, + 442847, + 442879, + 442911, + 442943, + 442975, + 443007, + 443039, + 443071, + 443103, + 443135, + 443167, + 443199, + 443231, + 443263, + 443295, + 443327, + 443359, + 443391, + 443423, + 443455, + 443487, + 443519, + 443551, + 443583, + 443615, + 443647, + 443679, + 443711, + 443743, + 443775, + 443807, + 443871, + 443903, + 443935, + 443967, + 443999, + 444031, + 444063, + 444095, + 444127, + 444159, + 444191, + 444223, + 444255, + 444287, + 444319, + 444351, + 444383, + 444415, + 444447, + 444479, + 444511, + 444543, + 444575, + 444607, + 444639, + 444671, + 444735, + 444767, + 444831, + 444863, + 444895, + 444927, + 444959, + 444991, + 445023, + 445055, + 445087, + 445119, + 445151, + 445183, + 445215, + 445247, + 445311, + 445343, + 445375, + 445407, + 445439, + 445471, + 445503, + 445535, + 445567, + 445599, + 445631, + 445663, + 445695, + 445727, + 445759, + 445791, + 445823, + 445855, + 445887, + 445919, + 445951, + 445983, + 446015, + 446047, + 446079, + 446111, + 446143, + 446175, + 446207, + 446239, + 446271, + 446303, + 446335, + 446367, + 446399, + 446431, + 446463, + 446495, + 446559, + 446591, + 446623, + 446655, + 446687, + 446719, + 446751, + 446783, + 446815, + 446847, + 446879, + 446911, + 446943, + 446975, + 447007, + 447039, + 447071, + 447103, + 447135, + 447167, + 447199, + 447231, + 447263, + 447295, + 447327, + 447359, + 447391, + 447455, + 447487, + 447519, + 447551, + 447583, + 447615, + 447647, + 447679, + 447743, + 447775, + 447807, + 447839, + 447871, + 447903, + 447935, + 447967, + 447999, + 448031, + 448063, + 448095, + 448127, + 448159, + 448223, + 448255, + 448287, + 448319, + 448351, + 448383, + 448415, + 448447, + 448479, + 448511, + 448543, + 448575, + 448639, + 448671, + 448703, + 448735, + 448767, + 448799, + 448831, + 448863, + 448895, + 448927, + 448959, + 448991, + 449023, + 449055, + 449087, + 449119, + 449151, + 449183, + 449215, + 449247, + 449279, + 449311, + 449343, + 449375, + 449439, + 449471, + 449503, + 449535, + 449567, + 449599, + 449631, + 449663, + 449695, + 449727, + 449759, + 449791, + 449823, + 449855, + 449887, + 449919, + 449951, + 449983, + 450015, + 450047, + 450079, + 450111, + 450143, + 450175, + 450207, + 450239, + 450271, + 450303, + 450335, + 450367, + 450399, + 450431, + 450463, + 450495, + 450559, + 450591, + 450623, + 450655, + 450687, + 450719, + 450751, + 450783, + 450815, + 450847, + 450879, + 450911, + 450943, + 450975, + 451007, + 451039, + 451071, + 451103, + 451135, + 451167, + 451199, + 451231, + 451263, + 451295, + 451327, + 451359, + 451391, + 451455, + 451487, + 451519, + 451551, + 451583, + 451615, + 451647, + 451679, + 451711, + 451743, + 451775, + 451807, + 451839, + 451871, + 451903, + 451935, + 451967, + 451999, + 452031, + 452063, + 452095, + 452127, + 452159, + 452191, + 452223, + 452255, + 452287, + 452319, + 452351, + 452383, + 452415, + 452447, + 452479, + 452511, + 452543, + 452575, + 452607, + 452639, + 452671, + 452703, + 452735, + 452767, + 452799, + 452831, + 452863, + 452895, + 452927, + 452959, + 453023, + 453055, + 453087, + 453119, + 453151, + 453183, + 453215, + 453247, + 453311, + 453343, + 453375, + 453407, + 453439, + 453471, + 453503, + 453535, + 453567, + 453599, + 453631, + 453663, + 453695, + 453727, + 453759, + 453791, + 453823, + 453855, + 453887, + 453919, + 453951, + 453983, + 454015, + 454047, + 454079, + 454111, + 454143, + 454207, + 454239, + 454271, + 454303, + 454335, + 454367, + 454399, + 454431, + 454463, + 454495, + 454527, + 454559, + 454591, + 454623, + 454655, + 454687, + 454719, + 454751, + 454783, + 454815, + 454847, + 454879, + 454911, + 454943, + 455007, + 455039, + 455071, + 455103, + 455135, + 455167, + 455199, + 455231, + 455327, + 455359, + 455391, + 455423, + 455455, + 455487, + 455519, + 455551, + 455583, + 455615, + 455647, + 455679, + 455711, + 455743, + 455775, + 455807, + 455839, + 455871, + 455903, + 455935, + 455967, + 455999, + 456031, + 456063, + 456095, + 456127, + 456191, + 456223, + 456255, + 456287, + 456319, + 456351, + 456383, + 456415, + 456447, + 456479, + 456511, + 456543, + 456575, + 456607, + 456639, + 456703, + 456735, + 456767, + 456799, + 456831, + 456863, + 456895, + 456927, + 456959, + 456991, + 457023, + 457055, + 457087, + 457119, + 457151, + 457183, + 457215, + 457247, + 457279, + 457311, + 457375, + 457407, + 457439, + 457471, + 457503, + 457535, + 457567, + 457631, + 457663, + 457727, + 457759, + 457791, + 457823, + 457855, + 457887, + 457919, + 457951, + 457983, + 458015, + 458047, + 458079, + 458111, + 458143, + 458175, + 458207, + 458239, + 458271, + 458303, + 458335, + 458367, + 458399, + 458431, + 458463, + 458495, + 458559, + 458591, + 458623, + 458655, + 458687, + 458719, + 458751, + 458815, + 458847, + 458879, + 458911, + 458975, + 459007, + 459039, + 459103, + 459135, + 459167, + 459199, + 459231, + 459263, + 459295, + 459327, + 459359, + 459391, + 459423, + 459455, + 459487, + 459519, + 459551, + 459583, + 459615, + 459647, + 459679, + 459711, + 459743, + 459775, + 459807, + 459871, + 459903, + 459935, + 459967, + 459999, + 460031, + 460063, + 460095, + 460127, + 460159, + 460191, + 460223, + 460255, + 460287, + 460319, + 460351, + 460383, + 460415, + 460447, + 460479, + 460511, + 460543, + 460575, + 460607, + 460671, + 460703, + 460735, + 460767, + 460799, + 460831, + 460863, + 460895, + 460927, + 460959, + 460991, + 461023, + 461055, + 461087, + 461119, + 461151, + 461183, + 461215, + 461247, + 461279, + 461311, + 461343, + 461375, + 461407, + 461439, + 461471, + 461503, + 461535, + 461567, + 461599, + 461631, + 461663, + 461695, + 461727, + 461759, + 461791, + 461823, + 461855, + 461887, + 461919, + 461951, + 461983, + 462015, + 462047, + 462079, + 462111, + 462143, + 462175, + 462207, + 462239, + 462271, + 462303, + 462335, + 462399, + 462431, + 462463, + 462495, + 462527, + 462559, + 462591, + 462623, + 462655, + 462687, + 462719, + 462751, + 462783, + 462815, + 462847, + 462879, + 462911, + 462943, + 462975, + 463007, + 463039, + 463071, + 463103, + 463135, + 463167, + 463199, + 463231, + 463263, + 463295, + 463327, + 463359, + 463391, + 463423, + 463455, + 463487, + 463519, + 463551, + 463583, + 463615, + 463647, + 463679, + 463711, + 463743, + 463775, + 463839, + 463871, + 463903, + 463935, + 463967, + 463999, + 464031, + 464063, + 464095, + 464127, + 464159, + 464191, + 464223, + 464255, + 464287, + 464319, + 464351, + 464383, + 464415, + 464447, + 464479, + 464511, + 464543, + 464575, + 464607, + 464639, + 464671, + 464703, + 464735, + 464767, + 464799, + 464831, + 464863, + 464895, + 464927, + 464959, + 464991, + 465023, + 465055, + 465087, + 465151, + 465183, + 465215, + 465247, + 465279, + 465311, + 465343, + 465375, + 465407, + 465439, + 465471, + 465503, + 465535, + 465567, + 465599, + 465631, + 465663, + 465695, + 465727, + 465759, + 465791, + 465823, + 465855, + 465887, + 465919, + 465951, + 465983, + 466015, + 466079, + 466111, + 466143, + 466175, + 466207, + 466239, + 466271, + 466303, + 466335, + 466399, + 466431, + 466463, + 466495, + 466527, + 466559, + 466591, + 466623, + 466655, + 466687, + 466719, + 466783, + 466815, + 466847, + 466879, + 466911, + 466943, + 466975, + 467007, + 467071, + 467103, + 467167, + 467199, + 467231, + 467263, + 467295, + 467327, + 467359, + 467391, + 467423, + 467455, + 467487, + 467519, + 467551, + 467583, + 467615, + 467647, + 467679, + 467711, + 467743, + 467775, + 467807, + 467839, + 467871, + 467903, + 467935, + 467967, + 467999, + 468031, + 468063, + 468095, + 468127, + 468159, + 468191, + 468223, + 468255, + 468287, + 468319, + 468351, + 468383, + 468415, + 468447, + 468479, + 468511, + 468543, + 468575, + 468607, + 468639, + 468671, + 468703, + 468735, + 468767, + 468799, + 468831, + 468863, + 468895, + 468927, + 468959, + 468991, + 469023, + 469055, + 469087, + 469119, + 469151, + 469183, + 469215, + 469247, + 469279, + 469311, + 469343, + 469375, + 469407, + 469439, + 469471, + 469503, + 469535, + 469567, + 469599, + 469631, + 469663, + 469695, + 469727, + 469759, + 469791, + 469823, + 469855, + 469887, + 469919, + 469951, + 469983, + 470015, + 470047, + 470079, + 470143, + 470175, + 470207, + 470239, + 470271, + 470303, + 470335, + 470367, + 470399, + 470431, + 470463, + 470495, + 470527, + 470559, + 470591, + 470623, + 470655, + 470687, + 470751, + 470783, + 470815, + 470847, + 470879, + 470911, + 470975, + 471007, + 471039, + 471071, + 471135, + 471167, + 471199, + 471231, + 471263, + 471327, + 471359, + 471423, + 471455, + 471487, + 471519, + 471551, + 471583, + 471615, + 471647, + 471679, + 471711, + 471743, + 471775, + 471807, + 471839, + 471871, + 471903, + 471935, + 471967, + 471999, + 472031, + 472063, + 472127, + 472159, + 472191, + 472223, + 472255, + 472287, + 472319, + 472351, + 472383, + 472415, + 472447, + 472479, + 472511, + 472543, + 472575, + 472607, + 472639, + 472671, + 472703, + 472735, + 472767, + 472799, + 472831, + 472863, + 472927, + 472959, + 472991, + 473023, + 473055, + 473119, + 473151, + 473183, + 473215, + 473247, + 473279, + 473311, + 473343, + 473375, + 473407, + 473439, + 473471, + 473503, + 473535, + 473567, + 473599, + 473631, + 473663, + 473695, + 473727, + 473759, + 473791, + 473823, + 473855, + 473887, + 473919, + 473951, + 473983, + 474015, + 474111, + 474143, + 474175, + 474207, + 474239, + 474271, + 474335, + 474367, + 474399, + 474431, + 474463, + 474495, + 474527, + 474559, + 474591, + 474623, + 474655, + 474719, + 474751, + 474783, + 474815, + 474847, + 474879, + 474911, + 474943, + 474975, + 475007, + 475039, + 475071, + 475103, + 475135, + 475167, + 475199, + 475231, + 475295, + 475327, + 475359, + 475391, + 475423, + 475455, + 475487, + 475519, + 475551, + 475583, + 475615, + 475647, + 475679, + 475711, + 475743, + 475807, + 475839, + 475871, + 475903, + 475935, + 475967, + 475999, + 476031, + 476063, + 476095, + 476127, + 476159, + 476191, + 476223, + 476255, + 476287, + 476319, + 476383, + 476415, + 476447, + 476479, + 476511, + 476543, + 476575, + 476607, + 476639, + 476671, + 476703, + 476735, + 476767, + 476799, + 476831, + 476863, + 476895, + 476927, + 476959, + 476991, + 477023, + 477055, + 477087, + 477119, + 477151, + 477183, + 477215, + 477247, + 477279, + 477311, + 477343, + 477375, + 477407, + 477439, + 477471, + 477503, + 477535, + 477567, + 477599, + 477631, + 477663, + 477695, + 477727, + 477759, + 477791, + 477823, + 477855, + 477887, + 477919, + 477951, + 478015, + 478047, + 478079, + 478111, + 478143, + 478175, + 478207, + 478239, + 478303, + 478335, + 478367, + 478399, + 478431, + 478463, + 478527, + 478559, + 478591, + 478623, + 478687, + 478719, + 478751, + 478783, + 478815, + 478847, + 478879, + 478911, + 478943, + 478975, + 479007, + 479039, + 479071, + 479103, + 479135, + 479167, + 479199, + 479231, + 479263, + 479327, + 479359, + 479391, + 479423, + 479455, + 479487, + 479519, + 479551, + 479583, + 479615, + 479647, + 479679, + 479711, + 479743, + 479775, + 479807, + 479839, + 479871, + 479903, + 479935, + 479967, + 479999, + 480031, + 480063, + 480095, + 480127, + 480159, + 480191, + 480223, + 480255, + 480287, + 480319, + 480351, + 480383, + 480415, + 480447, + 480479, + 480511, + 480543, + 480575, + 480607, + 480639, + 480671, + 480703, + 480735, + 480767, + 480831, + 480863, + 480895, + 480927, + 480959, + 480991, + 481023, + 481055, + 481087, + 481119, + 481151, + 481183, + 481215, + 481247, + 481279, + 481311, + 481343, + 481375, + 481471, + 481503, + 481535, + 481567, + 481599, + 481631, + 481695, + 481727, + 481759, + 481791, + 481823, + 481855, + 481887, + 481919, + 481951, + 481983, + 482015, + 482047, + 482079, + 482111, + 482143, + 482175, + 482207, + 482239, + 482303, + 482335, + 482367, + 482399, + 482431, + 482463, + 482495, + 482527, + 482559, + 482591, + 482623, + 482687, + 482719, + 482751, + 482783, + 482815, + 482847, + 482879, + 482911, + 482975, + 483007, + 483039, + 483071, + 483103, + 483135, + 483167, + 483199, + 483231, + 483263, + 483295, + 483327, + 483359, + 483391, + 483423, + 483455, + 483487, + 483519, + 483551, + 483583, + 483615, + 483647, + 483679, + 483711, + 483743, + 483775, + 483807, + 483839, + 483871, + 483903, + 483935, + 483967, + 483999, + 484031, + 484063, + 484095, + 484127, + 484159, + 484191, + 484223, + 484255, + 484287, + 484319, + 484351, + 484383, + 484415, + 484447, + 484479, + 484511, + 484543, + 484575, + 484607, + 484639, + 484671, + 484703, + 484735, + 484767, + 484799, + 484831, + 484863, + 484895, + 484927, + 484959, + 484991, + 485023, + 485087, + 485119, + 485151, + 485183, + 485215, + 485279, + 485311, + 485343, + 485375, + 485407, + 485439, + 485503, + 485535, + 485567, + 485599, + 485631, + 485663, + 485695, + 485727, + 485759, + 485791, + 485823, + 485855, + 485887, + 485919, + 485951, + 485983, + 486015, + 486079, + 486111, + 486143, + 486175, + 486207, + 486239, + 486271, + 486303, + 486367, + 486399, + 486431, + 486463, + 486495, + 486527, + 486559, + 486623, + 486655, + 486687, + 486719, + 486751, + 486783, + 486815, + 486847, + 486879, + 486911, + 486943, + 486975, + 487007, + 487039, + 487071, + 487103, + 487135, + 487167, + 487199, + 487231, + 487295, + 487327, + 487359, + 487391, + 487423, + 487455, + 487487, + 487519, + 487551, + 487583, + 487615, + 487647, + 487679, + 487711, + 487743, + 487775, + 487807, + 487839, + 487871, + 487903, + 487935, + 487967, + 487999, + 488031, + 488063, + 488095, + 488127, + 488159, + 488191, + 488223, + 488255, + 488319, + 488351, + 488383, + 488415, + 488447, + 488479, + 488511, + 488543, + 488575, + 488607, + 488639, + 488671, + 488703, + 488735, + 488767, + 488799, + 488831, + 488863, + 488895, + 488927, + 488959, + 488991, + 489023, + 489055, + 489087, + 489119, + 489151, + 489183, + 489215, + 489247, + 489279, + 489311, + 489343, + 489375, + 489407, + 489439, + 489471, + 489503, + 489535, + 489567, + 489599, + 489631, + 489663, + 489695, + 489727, + 489759, + 489791, + 489855, + 489919, + 489951, + 489983, + 490015, + 490047, + 490079, + 490111, + 490143, + 490207, + 490239, + 490271, + 490303, + 490335, + 490367, + 490399, + 490431, + 490463, + 490495, + 490527, + 490559, + 490591, + 490623, + 490655, + 490687, + 490719, + 490751, + 490783, + 490815, + 490847, + 490879, + 490911, + 490943, + 490975, + 491007, + 491039, + 491103, + 491135, + 491167, + 491199, + 491231, + 491263, + 491295, + 491327, + 491359, + 491391, + 491423, + 491455, + 491487, + 491551, + 491583, + 491615, + 491647, + 491679, + 491743, + 491775, + 491807, + 491839, + 491871, + 491903, + 491935, + 491967, + 491999, + 492031, + 492063, + 492095, + 492127, + 492159, + 492191, + 492255, + 492287, + 492319, + 492351, + 492383, + 492415, + 492447, + 492479, + 492511, + 492543, + 492575, + 492607, + 492639, + 492671, + 492703, + 492735, + 492767, + 492799, + 492831, + 492863, + 492895, + 492959, + 492991, + 493023, + 493055, + 493087, + 493119, + 493151, + 493183, + 493215, + 493247, + 493279, + 493311, + 493375, + 493407, + 493439, + 493471, + 493503, + 493535, + 493567, + 493599, + 493631, + 493663, + 493695, + 493727, + 493759, + 493791, + 493823, + 493855, + 493887, + 493919, + 493951, + 493983, + 494015, + 494047, + 494111, + 494143, + 494175, + 494207, + 494239, + 494271, + 494303, + 494335, + 494367, + 494399, + 494431, + 494463, + 494495, + 494559, + 494591, + 494623, + 494655, + 494687, + 494719, + 494751, + 494783, + 494815, + 494847, + 494879, + 494943, + 494975, + 495007, + 495039, + 495071, + 495103, + 495135, + 495167, + 495231, + 495263, + 495295, + 495327, + 495359, + 495391, + 495423, + 495455, + 495487, + 495519, + 495551, + 495583, + 495615, + 495647, + 495679, + 495711, + 495743, + 495775, + 495807, + 495839, + 495871, + 495903, + 495935, + 495967, + 495999, + 496031, + 496063, + 496095, + 496127, + 496159, + 496223, + 496255, + 496287, + 496319, + 496351, + 496383, + 496415, + 496447, + 496479, + 496511, + 496543, + 496575, + 496607, + 496639, + 496671, + 496703, + 496735, + 496767, + 496799, + 496831, + 496895, + 496927, + 496959, + 496991, + 497023, + 497055, + 497087, + 497119, + 497151, + 497183, + 497215, + 497247, + 497279, + 497311, + 497343, + 497375, + 497407, + 497439, + 497471, + 497503, + 497535, + 497567, + 497599, + 497631, + 497663, + 497695, + 497727, + 497759, + 497791, + 497855, + 497887, + 497919, + 497983, + 498015, + 498047, + 498079, + 498111, + 498143, + 498175, + 498207, + 498239, + 498271, + 498303, + 498335, + 498367, + 498399, + 498431, + 498463, + 498495, + 498527, + 498559, + 498591, + 498623, + 498655, + 498687, + 498719, + 498751, + 498783, + 498815, + 498847, + 498879, + 498911, + 498943, + 498975, + 499007, + 499039, + 499071, + 499135, + 499167, + 499199, + 499263, + 499295, + 499359, + 499391, + 499423, + 499455, + 499487, + 499519, + 499551, + 499583, + 499615, + 499647, + 499679, + 499711, + 499743, + 499775, + 499807, + 499839, + 499871, + 499903, + 499935, + 499999, + 500031, + 500063, + 500095, + 500127, + 500159, + 500191, + 500223, + 500255, + 500287, + 500319, + 500351, + 500383, + 500415, + 500447, + 500479, + 500511, + 500543, + 500575, + 500607, + 500639, + 500671, + 500703, + 500735, + 500767, + 500831, + 500863, + 500895, + 500959, + 500991, + 501023, + 501055, + 501087, + 501119, + 501151, + 501183, + 501215, + 501247, + 501279, + 501311, + 501343, + 501375, + 501407, + 501439, + 501471, + 501503, + 501535, + 501567, + 501599, + 501631, + 501695, + 501727, + 501759, + 501791, + 501823, + 501855, + 501887, + 501919, + 501951, + 501983, + 502015, + 502079, + 502111, + 502143, + 502175, + 502207, + 502239, + 502271, + 502303, + 502335, + 502367, + 502399, + 502431, + 502463, + 502495, + 502527, + 502559, + 502591, + 502623, + 502655, + 502687, + 502719, + 502751, + 502783, + 502815, + 502847, + 502879, + 502911, + 502943, + 502975, + 503007, + 503039, + 503071, + 503103, + 503167, + 503199, + 503231, + 503263, + 503295, + 503327, + 503359, + 503391, + 503423, + 503455, + 503487, + 503519, + 503551, + 503583, + 503615, + 503647, + 503679, + 503711, + 503743, + 503775, + 503807, + 503839, + 503871, + 503903, + 503935, + 503967, + 503999, + 504031, + 504063, + 504095, + 504127, + 504159, + 504191, + 504223, + 504255, + 504287, + 504319, + 504351, + 504383, + 504415, + 504447, + 504479, + 504511, + 504543, + 504575, + 504607, + 504639, + 504671, + 504703, + 504767, + 504799, + 504831, + 504863, + 504895, + 504927, + 504959, + 504991, + 505023, + 505055, + 505087, + 505119, + 505151, + 505183, + 505215, + 505247, + 505279, + 505311, + 505343, + 505375, + 505407, + 505439, + 505471, + 505503, + 505535, + 505567, + 505599, + 505631, + 505695, + 505727, + 505791, + 505823, + 505855, + 505887, + 505951, + 505983, + 506015, + 506047, + 506079, + 506111, + 506143, + 506175, + 506207, + 506239, + 506271, + 506303, + 506335, + 506367, + 506431, + 506463, + 506495, + 506527, + 506559, + 506591, + 506623, + 506655, + 506687, + 506719, + 506751, + 506783, + 506815, + 506847, + 506879, + 506911, + 506943, + 506975, + 507039, + 507103, + 507135, + 507167, + 507231, + 507263, + 507295, + 507327, + 507359, + 507391, + 507423, + 507455, + 507519, + 507551, + 507583, + 507647, + 507679, + 507711, + 507743, + 507775, + 507807, + 507839, + 507871, + 507903, + 507935, + 507967, + 507999, + 508031, + 508063, + 508095, + 508127, + 508159, + 508191, + 508223, + 508255, + 508287, + 508351, + 508383, + 508415, + 508447, + 508479, + 508511, + 508543, + 508575, + 508671, + 508703, + 508735, + 508767, + 508799, + 508831, + 508895, + 508927, + 508959, + 508991, + 509023, + 509055, + 509087, + 509119, + 509151, + 509183, + 509215, + 509247, + 509279, + 509311, + 509343, + 509407, + 509439, + 509471, + 509503, + 509535, + 509567, + 509599, + 509631, + 509663, + 509695, + 509727, + 509759, + 509791, + 509855, + 509887, + 509919, + 509951, + 509983, + 510015, + 510047, + 510079, + 510111, + 510143, + 510207, + 510239, + 510271, + 510303, + 510335, + 510399, + 510431, + 510463, + 510495, + 510527, + 510559, + 510591, + 510623, + 510655, + 510719, + 510751, + 510783, + 510815, + 510847, + 510879, + 510911, + 510943, + 510975, + 511039, + 511103, + 511167, + 511199, + 511231, + 511263, + 511295, + 511327, + 511391, + 511423, + 511455, + 511487, + 511551, + 511583, + 511615, + 511647, + 511679, + 511711, + 511743, + 511775, + 511839, + 511871, + 511903, + 511935, + 511967, + 512031, + 512095, + 512127, + 512159, + 512191, + 512223, + 512255, + 512319, + 512351, + 512383, + 512415, + 512447, + 512479, + 512511, + 512543, + 512575, + 512607, + 512639, + 512671, + 512735, + 512767, + 512799, + 512831, + 512863, + 512895, + 512959, + 512991, + 513023, + 513055, + 513087, + 513119, + 513151, + 513183, + 513215, + 513247, + 513279, + 513311, + 513343, + 513407, + 513439, + 513471, + 513503, + 513535, + 513567, + 513599, + 513663, + 513695, + 513727, + 513759, + 513791, + 513823, + 513855, + 513887, + 513919, + 513951, + 513983, + 514015, + 514047, + 514079, + 514111, + 514143, + 514239, + 514271, + 514303, + 514335, + 514367, + 514431, + 514463, + 514495, + 514527, + 514559, + 514591, + 514623, + 514655, + 514687, + 514719, + 514751, + 514783, + 514815, + 514847, + 514911, + 514943, + 514975, + 515007, + 515039, + 515071, + 515103, + 515135, + 515167, + 515199, + 515263, + 515295, + 515327, + 515359, + 515391, + 515423, + 515455, + 515487, + 515519, + 515551, + 515583, + 515647, + 515679, + 515711, + 515743, + 515775, + 515807, + 515839, + 515871, + 515903, + 515935, + 515967, + 515999, + 516031, + 516063, + 516095, + 516127, + 516159, + 516191, + 516223, + 516255, + 516319, + 516351, + 516383, + 516415, + 516479, + 516511, + 516543, + 516575, + 516607, + 516639, + 516671, + 516703, + 516799, + 516831, + 516895, + 516927, + 516959, + 516991, + 517055, + 517119, + 517151, + 517183, + 517215, + 517279, + 517311, + 517343, + 517375, + 517407, + 517439, + 517471, + 517503, + 517535, + 517567, + 517599, + 517631, + 517663, + 517727, + 517759, + 517791, + 517823, + 517855, + 517887, + 517919, + 517951, + 517983, + 518015, + 518047, + 518079, + 518111, + 518143, + 518175, + 518207, + 518239, + 518271, + 518303, + 518399, + 518431, + 518463, + 518495, + 518527, + 518559, + 518591, + 518623, + 518655, + 518687, + 518751, + 518783, + 518815, + 518847, + 518879, + 518911, + 518943, + 518975, + 519007, + 519039, + 519071, + 519103, + 519135, + 519167, + 519199, + 519231, + 519263, + 519295, + 519359, + 519391, + 519423, + 519455, + 519487, + 519583, + 519615, + 519647, + 519679, + 519711, + 519775, + 519807, + 519871, + 519935, + 519967, + 519999, + 520031, + 520063, + 520095, + 520127, + 520159, + 520191, + 520223, + 520255, + 520287, + 520319, + 520351, + 520383, + 520415, + 520479, + 520511, + 520543, + 520575, + 520607, + 520639, + 520735, + 520767, + 520799, + 520831, + 520863, + 520895, + 520927, + 520959, + 520991, + 521023, + 521055, + 521087, + 521119, + 521151, + 521183, + 521215, + 521247, + 521279, + 521311, + 521343, + 521375, + 521407, + 521439, + 521471, + 521503, + 521535, + 521567, + 521599, + 521631, + 521695, + 521727, + 521759, + 521791, + 521823, + 521855, + 521887, + 521919, + 521951, + 521983, + 522015, + 522047, + 522079, + 522111, + 522143, + 522175, + 522207, + 522239, + 522303, + 522335, + 522367, + 522399, + 522431, + 522463, + 522495, + 522559, + 522591, + 522623, + 522687, + 522719, + 522751, + 522783, + 522815, + 522847, + 522879, + 522911, + 522975, + 523007, + 523039, + 523071, + 523103, + 523135, + 523167, + 523199, + 523231, + 523263, + 523295, + 523327, + 523391, + 523423, + 523455, + 523487, + 523551, + 523583, + 523615, + 523679, + 523711, + 523743, + 523775, + 523839, + 523871, + 523935, + 524031, + 524063, + 524095, + 524127, + 524159, + 524191, + 524223, + 524255, + 524287, + 524351, + 524447, + 524479, + 524511, + 524543, + 524575, + 524607, + 524639, + 524671, + 524703, + 524735, + 524767, + 524799, + 524831, + 524863, + 524895, + 524927, + 524991, + 525023, + 525055, + 525087, + 525119, + 525151, + 525215, + 525247, + 525279, + 525311, + 525343, + 525375, + 525407, + 525439, + 525471, + 525503, + 525567, + 525599, + 525631, + 525663, + 525695, + 525727, + 525759, + 525791, + 525823, + 525855, + 525887, + 525919, + 525951, + 525983, + 526047, + 526079, + 526111, + 526143, + 526175, + 526239, + 526271, + 526335, + 526367, + 526431, + 526463, + 526495, + 526527, + 526591, + 526623, + 526655, + 526687, + 526719, + 526751, + 526783, + 526815, + 526847, + 526879, + 526911, + 526943, + 526975, + 527007, + 527039, + 527071, + 527135, + 527167, + 527199, + 527231, + 527263, + 527295, + 527327, + 527359, + 527391, + 527423, + 527455, + 527487, + 527519, + 527551, + 527583, + 527615, + 527679, + 527743, + 527775, + 527807, + 527839, + 527871, + 527903, + 527935, + 527967, + 527999, + 528095, + 528127, + 528159, + 528191, + 528223, + 528255, + 528287, + 528319, + 528351, + 528383, + 528415, + 528479, + 528511, + 528575, + 528607, + 528639, + 528703, + 528767, + 528799, + 528831, + 528863, + 528895, + 528927, + 528959, + 529055, + 529119, + 529151, + 529183, + 529215, + 529247, + 529279, + 529343, + 529375, + 529407, + 529439, + 529471, + 529503, + 529535, + 529567, + 529631, + 529727, + 529759, + 529823, + 529855, + 529887, + 529951, + 529983, + 530015, + 530047, + 530079, + 530111, + 530143, + 530175, + 530239, + 530335, + 530367, + 530431, + 530495, + 530527, + 530559, + 530591, + 530623, + 530687, + 530719, + 530751, + 530815, + 530847, + 530879, + 530943, + 530975, + 531007, + 531039, + 531071, + 531103, + 531135, + 531167, + 531199, + 531231, + 531263, + 531295, + 531327, + 531359, + 531423, + 531455, + 531487, + 531519, + 531615, + 531647, + 531679, + 531743, + 531775, + 531807, + 531839, + 531903, + 531935, + 531967, + 531999, + 532031, + 532063, + 532127, + 532159, + 532191, + 532255, + 532287, + 532319, + 532351, + 532383, + 532415, + 532447, + 532479, + 532543, + 532575, + 532607, + 532639, + 532671, + 532735, + 532767, + 532799, + 532895, + 532927, + 532959, + 532991, + 533023, + 533055, + 533087, + 533119, + 533151, + 533183, + 533215, + 533247, + 533279, + 533311, + 533343, + 533375, + 533407, + 533439, + 533471, + 533503, + 533535, + 533567, + 533599, + 533631, + 533695, + 533727, + 533759, + 533791, + 533823, + 533855, + 533887, + 533919, + 533951, + 533983, + 534015, + 534047, + 534079, + 534111, + 534143, + 534175, + 534207, + 534239, + 534271, + 534303, + 534335, + 534367, + 534399, + 534495, + 534527, + 534591, + 534623, + 534655, + 534687, + 534751, + 534815, + 534847, + 534879, + 534911, + 534943, + 534975, + 535007, + 535039, + 535071, + 535103, + 535135, + 535167, + 535199, + 535295, + 535327, + 535359, + 535391, + 535423, + 535455, + 535487, + 535519, + 535551, + 535583, + 535615, + 535647, + 535679, + 535711, + 535743, + 535775, + 535807, + 535839, + 535871, + 535935, + 535967, + 536031, + 536063, + 536127, + 536159, + 536191, + 536223, + 536255, + 536287, + 536319, + 536351, + 536383, + 536415, + 536447, + 536479, + 536543, + 536575, + 536639, + 536671, + 536703, + 536735, + 536767, + 536799, + 536831, + 536863, + 536895, + 536927, + 536959, + 536991, + 537055, + 537151, + 537183, + 537247, + 537279, + 537311, + 537343, + 537375, + 537407, + 537471, + 537503, + 537535, + 537567, + 537599, + 537631, + 537663, + 537695, + 537727, + 537759, + 537791, + 537823, + 537855, + 537887, + 537919, + 537951, + 537983, + 538015, + 538047, + 538079, + 538111, + 538143, + 538175, + 538207, + 538239, + 538271, + 538303, + 538367, + 538399, + 538431, + 538463, + 538495, + 538527, + 538559, + 538591, + 538623, + 538719, + 538751, + 538783, + 538815, + 538847, + 538879, + 538911, + 538943, + 538975, + 539007, + 539039, + 539071, + 539103, + 539135, + 539167, + 539199, + 539231, + 539263, + 539295, + 539327, + 539359, + 539391, + 539423, + 539487, + 539519, + 539551, + 539583, + 539615, + 539647, + 539679, + 539775, + 539807, + 539871, + 539935, + 539967, + 539999, + 540031, + 540063, + 540127, + 540159, + 540191, + 540223, + 540255, + 540287, + 540351, + 540383, + 540415, + 540447, + 540511, + 540543, + 540575, + 540639, + 540671, + 540703, + 540735, + 540767, + 540799, + 540831, + 540863, + 540895, + 540927, + 540959, + 540991, + 541023, + 541055, + 541087, + 541119, + 541215, + 541279, + 541311, + 541343, + 541375, + 541407, + 541439, + 541471, + 541503, + 541535, + 541599, + 541631, + 541663, + 541695, + 541727, + 541791, + 541823, + 541855, + 541887, + 541919, + 541951, + 541983, + 542015, + 542047, + 542079, + 542111, + 542143, + 542175, + 542207, + 542239, + 542271, + 542303, + 542335, + 542367, + 542399, + 542431, + 542463, + 542495, + 542527, + 542559, + 542591, + 542623, + 542655, + 542687, + 542719, + 542751, + 542783, + 542815, + 542847, + 542911, + 542943, + 542975, + 543071, + 543103, + 543167, + 543231, + 543263, + 543359, + 543391, + 543423, + 543455, + 543487, + 543519, + 543551, + 543615, + 543647, + 543679, + 543711, + 543743, + 543775, + 543807, + 543839, + 543871, + 543903, + 543935, + 543967, + 543999, + 544031, + 544063, + 544095, + 544127, + 544159, + 544191, + 544223, + 544255, + 544287, + 544319, + 544351, + 544383, + 544415, + 544447, + 544479, + 544511, + 544543, + 544575, + 544671, + 544703, + 544735, + 544767, + 544799, + 544831, + 544863, + 544927, + 544959, + 544991, + 545023, + 545055, + 545087, + 545119, + 545151, + 545183, + 545215, + 545247, + 545279, + 545311, + 545343, + 545375, + 545407, + 545439, + 545471, + 545503, + 545535, + 545567, + 545599, + 545631, + 545663, + 545695, + 545727, + 545823, + 545855, + 545887, + 545919, + 545951, + 545983, + 546015, + 546047, + 546111, + 546175, + 546207, + 546239, + 546271, + 546303, + 546367, + 546399, + 546431, + 546463, + 546495, + 546527, + 546623, + 546655, + 546687, + 546719, + 546751, + 546783, + 546815, + 546847, + 546879, + 546911, + 546975, + 547007, + 547039, + 547103, + 547135, + 547167, + 547199, + 547231, + 547263, + 547295, + 547359, + 547391, + 547423, + 547455, + 547487, + 547519, + 547551, + 547583, + 547647, + 547679, + 547711, + 547807, + 547839, + 547903, + 547935, + 547967, + 547999, + 548063, + 548127, + 548159, + 548191, + 548223, + 548255, + 548287, + 548319, + 548351, + 548383, + 548415, + 548447, + 548479, + 548511, + 548543, + 548575, + 548607, + 548639, + 548671, + 548703, + 548735, + 548767, + 548799, + 548831, + 548863, + 548895, + 548927, + 548959, + 548991, + 549023, + 549055, + 549119, + 549183, + 549215, + 549247, + 549279, + 549311, + 549343, + 549375, + 549407, + 549439, + 549471, + 549503, + 549535, + 549567, + 549599, + 549631, + 549663, + 549695, + 549727, + 549759, + 549791, + 549823, + 549855, + 549887, + 549919, + 549951, + 549983, + 550015, + 550047, + 550079, + 550111, + 550175, + 550239, + 550271, + 550303, + 550335, + 550399, + 550431, + 550463, + 550527, + 550559, + 550591, + 550623, + 550655, + 550751, + 550783, + 550815, + 550847, + 550879, + 550911, + 550943, + 550975, + 551007, + 551039, + 551071, + 551103, + 551135, + 551167, + 551199, + 551231, + 551263, + 551295, + 551359, + 551391, + 551423, + 551455, + 551487, + 551551, + 551583, + 551615, + 551647, + 551711, + 551743, + 551775, + 551807, + 551839, + 551871, + 551903, + 551935, + 551999, + 552031, + 552127, + 552159, + 552191, + 552223, + 552255, + 552287, + 552351, + 552383, + 552415, + 552447, + 552511, + 552543, + 552575, + 552607, + 552671, + 552703, + 552735, + 552767, + 552863, + 552895, + 552959, + 552991, + 553023, + 553055, + 553087, + 553119, + 553151, + 553183, + 553215, + 553247, + 553279, + 553311, + 553439, + 553471, + 553503, + 553535, + 553567, + 553599, + 553663, + 553727, + 553791, + 553823, + 553855, + 553887, + 553919, + 553951, + 553983, + 554015, + 554079, + 554111, + 554143, + 554175, + 554207, + 554239, + 554271, + 554335, + 554367, + 554399, + 554431, + 554463, + 554495, + 554527, + 554559, + 554591, + 554623, + 554655, + 554687, + 554719, + 554751, + 554783, + 554815, + 554847, + 554879, + 554911, + 554943, + 554975, + 555007, + 555039, + 555071, + 555103, + 555135, + 555167, + 555199, + 555231, + 555263, + 555327, + 555359, + 555391, + 555423, + 555455, + 555487, + 555519, + 555551, + 555583, + 555615, + 555647, + 555711, + 555743, + 555775, + 555807, + 555839, + 555871, + 555935, + 555967, + 555999, + 556031, + 556063, + 556159, + 556255, + 556287, + 556319, + 556351, + 556383, + 556415, + 556447, + 556479, + 556511, + 556575, + 556607, + 556639, + 556671, + 556703, + 556735, + 556767, + 556799, + 556831, + 556863, + 556927, + 556959, + 557023, + 557055, + 557087, + 557119, + 557151, + 557183, + 557215, + 557247, + 557279, + 557311, + 557343, + 557375, + 557407, + 557439, + 557471, + 557535, + 557567, + 557599, + 557631, + 557663, + 557695, + 557727, + 557759, + 557791, + 557823, + 557855, + 557887, + 557919, + 557951, + 557983, + 558015, + 558079, + 558111, + 558143, + 558175, + 558207, + 558271, + 558303, + 558335, + 558367, + 558399, + 558495, + 558527, + 558591, + 558623, + 558655, + 558687, + 558719, + 558751, + 558783, + 558815, + 558847, + 558879, + 558943, + 559039, + 559071, + 559103, + 559167, + 559199, + 559231, + 559263, + 559295, + 559327, + 559359, + 559391, + 559423, + 559455, + 559487, + 559519, + 559583, + 559615, + 559647, + 559679, + 559711, + 559775, + 559871, + 559903, + 559935, + 560031, + 560063, + 560095, + 560127, + 560191, + 560223, + 560255, + 560287, + 560351, + 560383, + 560415, + 560511, + 560575, + 560607, + 560639, + 560671, + 560735, + 560767, + 560799, + 560863, + 560895, + 560927, + 560959, + 560991, + 561023, + 561055, + 561087, + 561119, + 561151, + 561183, + 561215, + 561247, + 561279, + 561311, + 561343, + 561375, + 561407, + 561439, + 561471, + 561503, + 561535, + 561567, + 561599, + 561663, + 561727, + 561759, + 561791, + 561823, + 561855, + 561887, + 561919, + 561951, + 561983, + 562047, + 562111, + 562143, + 562175, + 562207, + 562271, + 562303, + 562335, + 562367, + 562399, + 562431, + 562495, + 562591, + 562623, + 562655, + 562687, + 562751, + 562783, + 562815, + 562847, + 562879, + 562911, + 562943, + 562975, + 563007, + 563039, + 563071, + 563103, + 563135, + 563167, + 563199, + 563231, + 563263, + 563295, + 563327, + 563359, + 563391, + 563423, + 563455, + 563487, + 563519, + 563551, + 563583, + 563647, + 563679, + 563711, + 563743, + 563807, + 563839, + 563871, + 563903, + 563935, + 563967, + 564031, + 564063, + 564095, + 564127, + 564159, + 564191, + 564223, + 564255, + 564287, + 564319, + 564351, + 564383, + 564415, + 564447, + 564543, + 564575, + 564639, + 564671, + 564703, + 564735, + 564799, + 564831, + 564863, + 564895, + 564927, + 564959, + 564991, + 565023, + 565055, + 565119, + 565151, + 565183, + 565247, + 565279, + 565311, + 565375, + 565407, + 565439, + 565503, + 565535, + 565567, + 565599, + 565631, + 565663, + 565695, + 565727, + 565759, + 565791, + 565823, + 565855, + 565887, + 565919, + 565951, + 565983, + 566015, + 566079, + 566111, + 566143, + 566175, + 566207, + 566239, + 566303, + 566335, + 566367, + 566399, + 566431, + 566463, + 566495, + 566527, + 566559, + 566591, + 566623, + 566655, + 566751, + 566783, + 566815, + 566847, + 566879, + 566911, + 566975, + 567007, + 567039, + 567071, + 567103, + 567135, + 567199, + 567263, + 567295, + 567327, + 567359, + 567391, + 567423, + 567455, + 567487, + 567519, + 567551, + 567583, + 567647, + 567679, + 567711, + 567743, + 567775, + 567839, + 567871, + 567935, + 567999, + 568031, + 568063, + 568095, + 568127, + 568159, + 568223, + 568255, + 568287, + 568319, + 568351, + 568383, + 568447, + 568479, + 568511, + 568543, + 568575, + 568607, + 568671, + 568703, + 568735, + 568767, + 568799, + 568863, + 568895, + 568927, + 568959, + 568991, + 569023, + 569087, + 569183, + 569215, + 569247, + 569279, + 569311, + 569343, + 569375, + 569407, + 569439, + 569471, + 569503, + 569535, + 569567, + 569599, + 569631, + 569663, + 569695, + 569759, + 569791, + 569823, + 569855, + 569887, + 569919, + 569951, + 569983, + 570015, + 570047, + 570079, + 570111, + 570143, + 570175, + 570239, + 570271, + 570303, + 570335, + 570367, + 570399, + 570431, + 570463, + 570527, + 570559, + 570623, + 570655, + 570687, + 570719, + 570751, + 570783, + 570815, + 570847, + 570911, + 570943, + 571071, + 571103, + 571135, + 571167, + 571199, + 571231, + 571263, + 571295, + 571327, + 571359, + 571391, + 571423, + 571455, + 571487, + 571551, + 571583, + 571615, + 571647, + 571679, + 571711, + 571743, + 571807, + 571871, + 571967, + 571999, + 572031, + 572063, + 572095, + 572127, + 572159, + 572191, + 572223, + 572287, + 572319, + 572351, + 572383, + 572415, + 572447, + 572511, + 572543, + 572575, + 572607, + 572639, + 572671, + 572767, + 572799, + 572831, + 572863, + 572895, + 572927, + 572959, + 572991, + 573023, + 573055, + 573087, + 573119, + 573151, + 573279, + 573311, + 573343, + 573375, + 573439, + 573471, + 573503, + 573535, + 573567, + 573599, + 573631, + 573663, + 573695, + 573727, + 573759, + 573791, + 573823, + 573855, + 573919, + 573951, + 573983, + 574015, + 574047, + 574079, + 574111, + 574143, + 574175, + 574207, + 574239, + 574271, + 574303, + 574335, + 574367, + 574399, + 574431, + 574463, + 574495, + 574527, + 574559, + 574591, + 574623, + 574655, + 574719, + 574751, + 574783, + 574815, + 574847, + 574879, + 574911, + 574943, + 574975, + 575007, + 575039, + 575071, + 575103, + 575135, + 575199, + 575231, + 575295, + 575327, + 575359, + 575391, + 575423, + 575455, + 575519, + 575551, + 575583, + 575615, + 575647, + 575679, + 575711, + 575743, + 575775, + 575807, + 575839, + 575871, + 575903, + 575935, + 575967, + 575999, + 576031, + 576063, + 576095, + 576127, + 576159, + 576191, + 576223, + 576255, + 576287, + 576383, + 576415, + 576447, + 576479, + 576511, + 576575, + 576607, + 576639, + 576671, + 576703, + 576735, + 576799, + 576831, + 576895, + 576927, + 576959, + 577023, + 577087, + 577119, + 577151, + 577183, + 577215, + 577247, + 577279, + 577311, + 577375, + 577407, + 577439, + 577471, + 577503, + 577535, + 577567, + 577599, + 577631, + 577663, + 577695, + 577855, + 577887, + 577919, + 577951, + 577983, + 578015, + 578047, + 578079, + 578143, + 578175, + 578207, + 578271, + 578335, + 578367, + 578399, + 578431, + 578463, + 578495, + 578527, + 578559, + 578623, + 578655, + 578687, + 578783, + 578815, + 578879, + 578911, + 578943, + 578975, + 579007, + 579039, + 579135, + 579167, + 579199, + 579231, + 579263, + 579295, + 579327, + 579359, + 579391, + 579423, + 579487, + 579519, + 579551, + 579583, + 579615, + 579647, + 579711, + 579743, + 579775, + 579807, + 579839, + 579871, + 579999, + 580095, + 580127, + 580159, + 580223, + 580255, + 580287, + 580319, + 580351, + 580383, + 580415, + 580447, + 580479, + 580511, + 580543, + 580575, + 580639, + 580671, + 580703, + 580767, + 580799, + 580831, + 580863, + 580895, + 580991, + 581023, + 581055, + 581087, + 581119, + 581151, + 581183, + 581215, + 581247, + 581279, + 581311, + 581343, + 581375, + 581407, + 581439, + 581471, + 581503, + 581535, + 581599, + 581631, + 581695, + 581727, + 581759, + 581823, + 581855, + 581887, + 581919, + 581951, + 581983, + 582015, + 582047, + 582079, + 582111, + 582143, + 582175, + 582271, + 582303, + 582367, + 582399, + 582431, + 582463, + 582495, + 582527, + 582559, + 582591, + 582623, + 582655, + 582687, + 582719, + 582783, + 582815, + 582847, + 582911, + 582943, + 582975, + 583007, + 583039, + 583071, + 583103, + 583135, + 583199, + 583231, + 583263, + 583295, + 583359, + 583391, + 583423, + 583455, + 583519, + 583551, + 583583, + 583647, + 583679, + 583743, + 583775, + 583807, + 583839, + 583871, + 583903, + 583935, + 583967, + 584031, + 584095, + 584191, + 584255, + 584319, + 584351, + 584415, + 584479, + 584511, + 584543, + 584575, + 584639, + 584671, + 584703, + 584735, + 584767, + 584799, + 584831, + 584863, + 584895, + 584927, + 584959, + 584991, + 585023, + 585055, + 585087, + 585151, + 585215, + 585247, + 585279, + 585311, + 585343, + 585375, + 585407, + 585471, + 585535, + 585567, + 585599, + 585631, + 585663, + 585695, + 585727, + 585759, + 585791, + 585823, + 585887, + 585951, + 585983, + 586015, + 586079, + 586111, + 586143, + 586207, + 586239, + 586271, + 586335, + 586367, + 586399, + 586463, + 586591, + 586623, + 586655, + 586687, + 586719, + 586751, + 586783, + 586847, + 586879, + 586911, + 586943, + 586975, + 587007, + 587039, + 587071, + 587103, + 587135, + 587167, + 587263, + 587327, + 587359, + 587391, + 587423, + 587455, + 587487, + 587519, + 587551, + 587583, + 587615, + 587647, + 587711, + 587743, + 587807, + 587871, + 587935, + 587967, + 587999, + 588031, + 588095, + 588127, + 588159, + 588191, + 588223, + 588255, + 588319, + 588351, + 588383, + 588415, + 588447, + 588479, + 588511, + 588543, + 588575, + 588607, + 588639, + 588671, + 588735, + 588767, + 588799, + 588831, + 588895, + 588927, + 588959, + 588991, + 589023, + 589055, + 589087, + 589119, + 589151, + 589183, + 589215, + 589279, + 589311, + 589343, + 589407, + 589471, + 589503, + 589535, + 589567, + 589599, + 589631, + 589663, + 589695, + 589727, + 589791, + 589823, + 589855, + 589887, + 589919, + 589951, + 589983, + 590015, + 590047, + 590111, + 590143, + 590207, + 590239, + 590271, + 590303, + 590335, + 590367, + 590399, + 590431, + 590463, + 590495, + 590527, + 590559, + 590591, + 590623, + 590655, + 590719, + 590751, + 590783, + 590815, + 590847, + 590879, + 590911, + 590943, + 590975, + 591007, + 591103, + 591135, + 591167, + 591199, + 591231, + 591327, + 591359, + 591423, + 591455, + 591519, + 591551, + 591583, + 591615, + 591647, + 591679, + 591775, + 591839, + 591871, + 591903, + 591935, + 591999, + 592063, + 592095, + 592127, + 592159, + 592223, + 592255, + 592287, + 592319, + 592351, + 592383, + 592415, + 592447, + 592479, + 592575, + 592639, + 592735, + 592767, + 592799, + 592831, + 592863, + 592895, + 592927, + 592959, + 592991, + 593055, + 593151, + 593183, + 593215, + 593247, + 593343, + 593375, + 593407, + 593439, + 593471, + 593535, + 593567, + 593631, + 593663, + 593695, + 593727, + 593759, + 593791, + 593823, + 593855, + 593887, + 593951, + 594015, + 594047, + 594079, + 594111, + 594175, + 594207, + 594239, + 594271, + 594335, + 594367, + 594399, + 594431, + 594463, + 594495, + 594527, + 594559, + 594591, + 594623, + 594687, + 594719, + 594751, + 594815, + 594847, + 594879, + 594911, + 594943, + 594975, + 595007, + 595039, + 595071, + 595167, + 595199, + 595231, + 595263, + 595295, + 595327, + 595359, + 595391, + 595423, + 595455, + 595487, + 595519, + 595551, + 595583, + 595615, + 595647, + 595679, + 595711, + 595743, + 595775, + 595807, + 595839, + 595871, + 595903, + 595935, + 595967, + 596031, + 596127, + 596159, + 596191, + 596287, + 596319, + 596383, + 596415, + 596447, + 596479, + 596511, + 596543, + 596575, + 596607, + 596639, + 596671, + 596735, + 596767, + 596799, + 596831, + 596863, + 596895, + 596927, + 596991, + 597023, + 597055, + 597087, + 597119, + 597183, + 597215, + 597247, + 597279, + 597311, + 597343, + 597407, + 597439, + 597503, + 597567, + 597599, + 597631, + 597663, + 597695, + 597727, + 597791, + 597823, + 597855, + 597887, + 597919, + 597951, + 597983, + 598015, + 598047, + 598079, + 598143, + 598207, + 598239, + 598335, + 598399, + 598431, + 598463, + 598527, + 598559, + 598623, + 598655, + 598687, + 598719, + 598783, + 598879, + 598911, + 598943, + 598975, + 599007, + 599039, + 599071, + 599103, + 599135, + 599167, + 599199, + 599231, + 599263, + 599295, + 599391, + 599423, + 599487, + 599551, + 599583, + 599615, + 599647, + 599679, + 599743, + 599775, + 599807, + 599839, + 599871, + 599999, + 600031, + 600127, + 600159, + 600223, + 600255, + 600287, + 600319, + 600351, + 600383, + 600415, + 600447, + 600479, + 600511, + 600543, + 600575, + 600607, + 600639, + 600703, + 600735, + 600767, + 600799, + 600831, + 600863, + 600895, + 600959, + 601023, + 601087, + 601119, + 601151, + 601215, + 601247, + 601279, + 601311, + 601375, + 601407, + 601439, + 601471, + 601503, + 601535, + 601567, + 601599, + 601631, + 601663, + 601695, + 601727, + 601759, + 601791, + 601855, + 601887, + 601919, + 601951, + 601983, + 602015, + 602047, + 602079, + 602111, + 602143, + 602175, + 602207, + 602239, + 602303, + 602335, + 602367, + 602399, + 602431, + 602463, + 602495, + 602527, + 602559, + 602591, + 602623, + 602655, + 602687, + 602719, + 602751, + 602783, + 602815, + 602879, + 602943, + 602975, + 603039, + 603071, + 603103, + 603135, + 603167, + 603231, + 603263, + 603295, + 603327, + 603391, + 603423, + 603455, + 603487, + 603551, + 603615, + 603647, + 603679, + 603711, + 603743, + 603775, + 603807, + 603839, + 603903, + 603935, + 603967, + 603999, + 604031, + 604063, + 604095, + 604127, + 604159, + 604223, + 604255, + 604287, + 604319, + 604351, + 604415, + 604447, + 604479, + 604511, + 604607, + 604639, + 604671, + 604703, + 604735, + 604767, + 604831, + 604927, + 604959, + 604991, + 605023, + 605055, + 605087, + 605119, + 605151, + 605183, + 605215, + 605247, + 605279, + 605311, + 605343, + 605375, + 605407, + 605471, + 605503, + 605535, + 605631, + 605663, + 605759, + 605823, + 605855, + 605887, + 605919, + 605951, + 606015, + 606047, + 606079, + 606111, + 606143, + 606271, + 606303, + 606335, + 606367, + 606399, + 606431, + 606559, + 606623, + 606655, + 606687, + 606719, + 606751, + 606783, + 606815, + 606847, + 606975, + 607007, + 607071, + 607103, + 607135, + 607167, + 607231, + 607263, + 607295, + 607327, + 607359, + 607391, + 607455, + 607487, + 607519, + 607551, + 607615, + 607647, + 607679, + 607743, + 607775, + 607807, + 607871, + 607903, + 607967, + 607999, + 608063, + 608095, + 608127, + 608191, + 608223, + 608255, + 608287, + 608319, + 608351, + 608383, + 608415, + 608447, + 608479, + 608511, + 608543, + 608575, + 608607, + 608639, + 608671, + 608703, + 608735, + 608799, + 608831, + 608927, + 608959, + 609023, + 609055, + 609119, + 609151, + 609183, + 609215, + 609247, + 609311, + 609375, + 609407, + 609439, + 609471, + 609503, + 609535, + 609567, + 609599, + 609631, + 609663, + 609695, + 609727, + 609759, + 609791, + 609823, + 609855, + 609887, + 609951, + 609983, + 610015, + 610047, + 610079, + 610111, + 610143, + 610175, + 610207, + 610303, + 610335, + 610367, + 610399, + 610431, + 610463, + 610495, + 610559, + 610591, + 610623, + 610687, + 610719, + 610783, + 610815, + 610847, + 610879, + 610911, + 610943, + 610975, + 611007, + 611039, + 611071, + 611103, + 611199, + 611231, + 611295, + 611327, + 611359, + 611391, + 611423, + 611455, + 611487, + 611519, + 611551, + 611583, + 611615, + 611679, + 611711, + 611743, + 611839, + 611871, + 611903, + 611999, + 612127, + 612191, + 612255, + 612287, + 612319, + 612383, + 612415, + 612447, + 612511, + 612575, + 612607, + 612671, + 612703, + 612735, + 612799, + 612831, + 612863, + 612895, + 612927, + 612959, + 613023, + 613055, + 613087, + 613119, + 613151, + 613183, + 613247, + 613279, + 613375, + 613407, + 613471, + 613503, + 613567, + 613599, + 613631, + 613663, + 613695, + 613727, + 613759, + 613823, + 613855, + 613887, + 613919, + 613951, + 614015, + 614079, + 614111, + 614143, + 614175, + 614303, + 614367, + 614399, + 614431, + 614463, + 614495, + 614527, + 614559, + 614623, + 614655, + 614719, + 614751, + 614783, + 614815, + 614847, + 614879, + 614911, + 614943, + 615007, + 615039, + 615071, + 615135, + 615167, + 615231, + 615263, + 615295, + 615327, + 615359, + 615391, + 615423, + 615455, + 615487, + 615583, + 615647, + 615743, + 615775, + 615807, + 615839, + 615871, + 615935, + 615967, + 616031, + 616063, + 616095, + 616127, + 616191, + 616223, + 616287, + 616319, + 616383, + 616415, + 616447, + 616479, + 616607, + 616671, + 616703, + 616735, + 616831, + 616863, + 616895, + 616927, + 616959, + 616991, + 617087, + 617119, + 617151, + 617215, + 617247, + 617279, + 617311, + 617375, + 617407, + 617439, + 617503, + 617535, + 617567, + 617599, + 617631, + 617663, + 617695, + 617727, + 617759, + 617791, + 617823, + 617855, + 617887, + 617983, + 618015, + 618047, + 618207, + 618271, + 618335, + 618367, + 618399, + 618431, + 618495, + 618527, + 618591, + 618655, + 618687, + 618719, + 618751, + 618815, + 618847, + 618911, + 618943, + 618975, + 619007, + 619039, + 619071, + 619135, + 619167, + 619199, + 619231, + 619263, + 619295, + 619327, + 619359, + 619391, + 619423, + 619455, + 619487, + 619551, + 619583, + 619647, + 619679, + 619711, + 619743, + 619775, + 619807, + 619839, + 619871, + 619903, + 619935, + 619967, + 620031, + 620063, + 620095, + 620159, + 620223, + 620255, + 620287, + 620319, + 620351, + 620383, + 620415, + 620447, + 620479, + 620511, + 620543, + 620575, + 620607, + 620671, + 620703, + 620735, + 620767, + 620831, + 620927, + 620959, + 621023, + 621055, + 621087, + 621119, + 621151, + 621183, + 621215, + 621279, + 621311, + 621343, + 621375, + 621439, + 621471, + 621503, + 621535, + 621567, + 621599, + 621727, + 621759, + 621791, + 621823, + 621855, + 621919, + 621951, + 621983, + 622015, + 622047, + 622079, + 622175, + 622207, + 622303, + 622335, + 622367, + 622399, + 622463, + 622495, + 622559, + 622591, + 622687, + 622751, + 622783, + 622847, + 622911, + 622943, + 623007, + 623039, + 623071, + 623103, + 623167, + 623231, + 623263, + 623295, + 623327, + 623359, + 623391, + 623423, + 623487, + 623551, + 623583, + 623615, + 623647, + 623679, + 623743, + 623775, + 623807, + 623839, + 623903, + 623935, + 623967, + 623999, + 624031, + 624063, + 624127, + 624159, + 624191, + 624255, + 624287, + 624319, + 624351, + 624383, + 624415, + 624447, + 624479, + 624511, + 624543, + 624575, + 624607, + 624639, + 624671, + 624735, + 624767, + 624831, + 624895, + 624927, + 624991, + 625023, + 625087, + 625119, + 625151, + 625183, + 625215, + 625279, + 625311, + 625343, + 625471, + 625503, + 625567, + 625599, + 625631, + 625663, + 625695, + 625759, + 625823, + 625887, + 625919, + 625951, + 626015, + 626079, + 626143, + 626175, + 626271, + 626303, + 626335, + 626367, + 626399, + 626431, + 626463, + 626495, + 626527, + 626623, + 626655, + 626687, + 626783, + 626815, + 626847, + 626879, + 626911, + 626943, + 627007, + 627071, + 627135, + 627199, + 627231, + 627263, + 627359, + 627391, + 627487, + 627519, + 627615, + 627647, + 627679, + 627711, + 627775, + 627871, + 628095, + 628159, + 628191, + 628223, + 628255, + 628319, + 628351, + 628383, + 628575, + 628607, + 628671, + 628703, + 628767, + 628799, + 628831, + 628863, + 628895, + 628927, + 628959, + 629087, + 629119, + 629151, + 629247, + 629279, + 629311, + 629375, + 629439, + 629503, + 629535, + 629567, + 629599, + 629631, + 629663, + 629695, + 629727, + 629759, + 629823, + 629855, + 629919, + 630015, + 630079, + 630111, + 630143, + 630207, + 630271, + 630303, + 630335, + 630367, + 630399, + 630431, + 630495, + 630527, + 630591, + 630623, + 630655, + 630687, + 630719, + 630815, + 630879, + 630943, + 630975, + 631039, + 631071, + 631103, + 631167, + 631199, + 631231, + 631263, + 631295, + 631327, + 631391, + 631455, + 631487, + 631583, + 631647, + 631679, + 631775, + 631807, + 631839, + 631871, + 631903, + 631935, + 631967, + 631999, + 632063, + 632095, + 632127, + 632191, + 632255, + 632287, + 632319, + 632351, + 632383, + 632415, + 632447, + 632479, + 632511, + 632543, + 632575, + 632607, + 632639, + 632671, + 632703, + 632735, + 632767, + 632799, + 632863, + 632895, + 632927, + 632959, + 632991, + 633023, + 633087, + 633119, + 633151, + 633183, + 633215, + 633247, + 633279, + 633311, + 633343, + 633375, + 633407, + 633439, + 633471, + 633503, + 633535, + 633567, + 633599, + 633631, + 633695, + 633727, + 633759, + 633823, + 633855, + 633983, + 634111, + 634143, + 634175, + 634207, + 634239, + 634271, + 634303, + 634335, + 634463, + 634495, + 634527, + 634559, + 634591, + 634655, + 634687, + 634719, + 634751, + 634783, + 634847, + 634911, + 634943, + 635007, + 635039, + 635071, + 635135, + 635167, + 635263, + 635327, + 635391, + 635487, + 635583, + 635647, + 635679, + 635711, + 635775, + 635807, + 635839, + 635903, + 635935, + 635967, + 635999, + 636031, + 636063, + 636095, + 636127, + 636159, + 636223, + 636255, + 636319, + 636351, + 636383, + 636415, + 636479, + 636511, + 636543, + 636575, + 636671, + 636735, + 636831, + 636863, + 636927, + 636991, + 637023, + 637055, + 637087, + 637183, + 637215, + 637247, + 637279, + 637311, + 637343, + 637375, + 637471, + 637599, + 637695, + 637727, + 637759, + 637823, + 637855, + 637951, + 637983, + 638015, + 638047, + 638111, + 638143, + 638175, + 638207, + 638239, + 638335, + 638367, + 638399, + 638431, + 638463, + 638495, + 638527, + 638591, + 638623, + 638655, + 638719, + 638751, + 638783, + 638815, + 638847, + 638879, + 639039, + 639071, + 639103, + 639135, + 639167, + 639231, + 639295, + 639327, + 639359, + 639423, + 639455, + 639487, + 639519, + 639583, + 639615, + 639679, + 639711, + 639775, + 639807, + 639871, + 639903, + 639935, + 639967, + 640031, + 640063, + 640095, + 640127, + 640159, + 640191, + 640223, + 640255, + 640287, + 640319, + 640351, + 640415, + 640447, + 640511, + 640543, + 640575, + 640607, + 640639, + 640703, + 640735, + 640767, + 640799, + 640863, + 640895, + 640927, + 640959, + 640991, + 641023, + 641055, + 641087, + 641151, + 641183, + 641215, + 641311, + 641343, + 641375, + 641407, + 641439, + 641471, + 641503, + 641535, + 641663, + 641727, + 641759, + 641823, + 641855, + 641887, + 641919, + 642015, + 642047, + 642079, + 642111, + 642143, + 642239, + 642335, + 642367, + 642431, + 642495, + 642527, + 642559, + 642591, + 642623, + 642655, + 642719, + 642751, + 642783, + 642815, + 642847, + 642879, + 642911, + 643007, + 643039, + 643103, + 643135, + 643199, + 643231, + 643263, + 643359, + 643391, + 643423, + 643455, + 643519, + 643551, + 643583, + 643615, + 643647, + 643775, + 643807, + 643871, + 643935, + 643967, + 643999, + 644031, + 644063, + 644095, + 644191, + 644223, + 644255, + 644319, + 644383, + 644415, + 644447, + 644479, + 644511, + 644575, + 644607, + 644831, + 644895, + 644959, + 645023, + 645055, + 645087, + 645183, + 645247, + 645279, + 645311, + 645343, + 645375, + 645407, + 645439, + 645503, + 645535, + 645567, + 645599, + 645631, + 645663, + 645695, + 645727, + 645823, + 645855, + 645887, + 645919, + 645951, + 645983, + 646047, + 646079, + 646111, + 646143, + 646175, + 646207, + 646367, + 646399, + 646431, + 646463, + 646527, + 646559, + 646591, + 646623, + 646655, + 646687, + 646815, + 646879, + 646911, + 647007, + 647071, + 647103, + 647167, + 647199, + 647231, + 647423, + 647455, + 647487, + 647519, + 647583, + 647615, + 647647, + 647679, + 647711, + 647743, + 647775, + 647807, + 647903, + 647935, + 647967, + 647999, + 648095, + 648127, + 648191, + 648255, + 648319, + 648351, + 648383, + 648415, + 648479, + 648511, + 648575, + 648607, + 648639, + 648703, + 648735, + 648767, + 648895, + 648927, + 648991, + 649023, + 649087, + 649119, + 649151, + 649183, + 649215, + 649247, + 649279, + 649311, + 649343, + 649375, + 649407, + 649471, + 649503, + 649567, + 649599, + 649663, + 649695, + 649727, + 649759, + 649791, + 649887, + 649919, + 649983, + 650015, + 650047, + 650079, + 650175, + 650207, + 650239, + 650271, + 650303, + 650335, + 650431, + 650463, + 650495, + 650527, + 650559, + 650591, + 650687, + 650751, + 650815, + 650847, + 650879, + 650911, + 650943, + 651007, + 651071, + 651103, + 651135, + 651199, + 651231, + 651263, + 651295, + 651327, + 651391, + 651423, + 651455, + 651487, + 651519, + 651551, + 651583, + 651615, + 651647, + 651679, + 651711, + 651743, + 651775, + 651807, + 651839, + 651871, + 651903, + 651935, + 651967, + 651999, + 652031, + 652095, + 652159, + 652223, + 652255, + 652351, + 652383, + 652447, + 652479, + 652575, + 652607, + 652671, + 652703, + 652799, + 652831, + 652863, + 652895, + 652927, + 652959, + 652991, + 653023, + 653087, + 653119, + 653183, + 653247, + 653279, + 653439, + 653503, + 653567, + 653631, + 653759, + 653823, + 653855, + 653919, + 653951, + 653983, + 654015, + 654047, + 654143, + 654175, + 654239, + 654463, + 654495, + 654527, + 654559, + 654591, + 654623, + 654655, + 654687, + 654719, + 654751, + 654783, + 654815, + 654879, + 654911, + 654943, + 654975, + 655007, + 655039, + 655103, + 655167, + 655359, + 655391, + 655455, + 655551, + 655583, + 655615, + 655647, + 655679, + 655711, + 655743, + 655775, + 655807, + 655839, + 655871, + 655903, + 655935, + 655999, + 656095, + 656127, + 656159, + 656191, + 656255, + 656287, + 656319, + 656351, + 656383, + 656511, + 656543, + 656671, + 656703, + 656767, + 656799, + 656863, + 656927, + 656991, + 657055, + 657119, + 657183, + 657247, + 657343, + 657375, + 657471, + 657503, + 657535, + 657567, + 657599, + 657631, + 657663, + 657695, + 657727, + 657759, + 657791, + 657823, + 657855, + 657919, + 657951, + 658015, + 658079, + 658111, + 658143, + 658207, + 658271, + 658303, + 658367, + 658399, + 658463, + 658591, + 658623, + 658655, + 658687, + 658719, + 658751, + 658783, + 658847, + 658943, + 659039, + 659071, + 659103, + 659135, + 659167, + 659199, + 659263, + 659295, + 659359, + 659519, + 659551, + 659679, + 659711, + 659743, + 659935, + 659967, + 659999, + 660031, + 660063, + 660095, + 660127, + 660223, + 660319, + 660351, + 660383, + 660415, + 660447, + 660479, + 660543, + 660575, + 660639, + 660671, + 660863, + 660959, + 660991, + 661087, + 661119, + 661151, + 661215, + 661279, + 661311, + 661375, + 661439, + 661471, + 661503, + 661535, + 661567, + 661599, + 661631, + 661823, + 661855, + 661887, + 661919, + 661951, + 662047, + 662079, + 662143, + 662207, + 662239, + 662271, + 662303, + 662335, + 662367, + 662495, + 662527, + 662559, + 662591, + 662623, + 662687, + 662751, + 662815, + 662911, + 662975, + 663007, + 663071, + 663103, + 663135, + 663167, + 663199, + 663231, + 663263, + 663295, + 663327, + 663359, + 663423, + 663455, + 663551, + 663583, + 663615, + 663647, + 663679, + 663711, + 663775, + 663839, + 663903, + 663967, + 663999, + 664031, + 664095, + 664127, + 664159, + 664191, + 664223, + 664255, + 664287, + 664319, + 664351, + 664383, + 664511, + 664607, + 664671, + 664703, + 664767, + 664831, + 664895, + 664959, + 664991, + 665055, + 665087, + 665183, + 665247, + 665311, + 665375, + 665407, + 665439, + 665599, + 665727, + 665759, + 665791, + 665823, + 665887, + 665951, + 666015, + 666047, + 666079, + 666111, + 666143, + 666207, + 666239, + 666271, + 666335, + 666367, + 666431, + 666463, + 666495, + 666591, + 666655, + 666719, + 666751, + 666783, + 666815, + 666847, + 666879, + 666911, + 667039, + 667071, + 667103, + 667135, + 667199, + 667231, + 667263, + 667295, + 667391, + 667423, + 667455, + 667551, + 667583, + 667615, + 667711, + 667743, + 667775, + 667807, + 667839, + 667871, + 667903, + 667935, + 667967, + 667999, + 668095, + 668127, + 668159, + 668191, + 668223, + 668255, + 668479, + 668511, + 668543, + 668575, + 668607, + 668639, + 668671, + 668735, + 668767, + 668799, + 668831, + 668895, + 668927, + 668959, + 668991, + 669055, + 669087, + 669151, + 669247, + 669279, + 669311, + 669343, + 669375, + 669439, + 669503, + 669599, + 669695, + 669727, + 669759, + 669791, + 669823, + 669855, + 669887, + 669951, + 669983, + 670047, + 670079, + 670111, + 670143, + 670207, + 670239, + 670271, + 670303, + 670335, + 670463, + 670559, + 670591, + 670623, + 670655, + 670687, + 670751, + 670815, + 670847, + 670879, + 670911, + 670943, + 670975, + 671007, + 671039, + 671071, + 671103, + 671199, + 671231, + 671263, + 671359, + 671455, + 671487, + 671519, + 671551, + 671743, + 671775, + 671807, + 671839, + 671871, + 671935, + 671967, + 671999, + 672031, + 672063, + 672095, + 672159, + 672191, + 672255, + 672287, + 672319, + 672351, + 672575, + 672607, + 672671, + 672799, + 672863, + 672895, + 672959, + 673087, + 673151, + 673183, + 673247, + 673343, + 673375, + 673407, + 673439, + 673471, + 673503, + 673535, + 673567, + 673631, + 673695, + 673727, + 673759, + 673791, + 673823, + 673887, + 673951, + 674015, + 674047, + 674111, + 674239, + 674335, + 674367, + 674431, + 674463, + 674495, + 674591, + 674751, + 674783, + 674815, + 674879, + 674911, + 674943, + 674975, + 675007, + 675039, + 675071, + 675103, + 675167, + 675199, + 675231, + 675327, + 675423, + 675519, + 675551, + 675583, + 675711, + 675743, + 675775, + 675871, + 675903, + 675935, + 675967, + 676031, + 676063, + 676095, + 676127, + 676191, + 676255, + 676319, + 676351, + 676383, + 676415, + 676447, + 676511, + 676543, + 676575, + 676671, + 676703, + 676735, + 676799, + 676959, + 677023, + 677055, + 677087, + 677183, + 677215, + 677247, + 677279, + 677311, + 677343, + 677407, + 677439, + 677471, + 677535, + 677599, + 677663, + 677695, + 677855, + 677887, + 677951, + 677983, + 678175, + 678271, + 678303, + 678367, + 678399, + 678463, + 678495, + 678559, + 678623, + 678655, + 678687, + 678719, + 678847, + 678879, + 678911, + 678943, + 678975, + 679039, + 679071, + 679135, + 679167, + 679199, + 679263, + 679295, + 679327, + 679359, + 679423, + 679487, + 679615, + 679647, + 679679, + 679711, + 679743, + 679775, + 679807, + 679871, + 679935, + 679967, + 680031, + 680063, + 680095, + 680159, + 680223, + 680255, + 680287, + 680319, + 680351, + 680447, + 680511, + 680543, + 680575, + 680607, + 680735, + 680767, + 680799, + 680831, + 680863, + 680895, + 680991, + 681119, + 681183, + 681215, + 681247, + 681279, + 681311, + 681375, + 681407, + 681439, + 681471, + 681567, + 681599, + 681695, + 681759, + 681791, + 681887, + 681919, + 681983, + 682079, + 682111, + 682207, + 682239, + 682271, + 682335, + 682367, + 682399, + 682431, + 682463, + 682495, + 682527, + 682559, + 682591, + 682687, + 682751, + 682783, + 682815, + 682847, + 682911, + 682943, + 682975, + 683039, + 683071, + 683103, + 683135, + 683199, + 683327, + 683519, + 683551, + 683583, + 683647, + 683679, + 683711, + 683743, + 683807, + 683839, + 683871, + 683903, + 683967, + 683999, + 684031, + 684063, + 684095, + 684159, + 684223, + 684255, + 684287, + 684319, + 684351, + 684479, + 684511, + 684543, + 684575, + 684607, + 684639, + 684671, + 684703, + 684799, + 684863, + 684895, + 684927, + 684959, + 685023, + 685087, + 685183, + 685215, + 685247, + 685311, + 685375, + 685439, + 685503, + 685535, + 685599, + 685631, + 685663, + 685727, + 685759, + 685791, + 685823, + 685887, + 685919, + 685951, + 685983, + 686047, + 686111, + 686143, + 686175, + 686239, + 686303, + 686367, + 686431, + 686463, + 686623, + 686655, + 686751, + 686783, + 686847, + 686879, + 686943, + 686975, + 687007, + 687039, + 687199, + 687231, + 687263, + 687295, + 687327, + 687359, + 687423, + 687455, + 687487, + 687551, + 687615, + 687647, + 687775, + 687807, + 687839, + 687871, + 687935, + 687999, + 688095, + 688127, + 688223, + 688383, + 688415, + 688447, + 688511, + 688543, + 688575, + 688607, + 688671, + 688703, + 688767, + 688831, + 688863, + 688927, + 688959, + 688991, + 689023, + 689055, + 689087, + 689151, + 689311, + 689343, + 689375, + 689407, + 689439, + 689503, + 689599, + 689663, + 689695, + 689791, + 689855, + 689887, + 689919, + 689951, + 689983, + 690015, + 690175, + 690239, + 690271, + 690303, + 690335, + 690367, + 690399, + 690431, + 690463, + 690527, + 690623, + 690655, + 690751, + 690847, + 690911, + 690943, + 691007, + 691039, + 691071, + 691103, + 691167, + 691199, + 691295, + 691327, + 691423, + 691519, + 691551, + 691615, + 691711, + 691775, + 691871, + 691967, + 692063, + 692095, + 692223, + 692255, + 692287, + 692351, + 692415, + 692479, + 692511, + 692575, + 692639, + 692703, + 692767, + 692799, + 692863, + 692895, + 692959, + 692991, + 693055, + 693215, + 693279, + 693311, + 693343, + 693375, + 693407, + 693439, + 693535, + 693567, + 693599, + 693631, + 693663, + 693759, + 693791, + 693823, + 693887, + 693919, + 694015, + 694079, + 694111, + 694143, + 694175, + 694367, + 694399, + 694431, + 694495, + 694527, + 694559, + 694623, + 694687, + 694879, + 694911, + 695039, + 695071, + 695103, + 695135, + 695199, + 695295, + 695359, + 695391, + 695423, + 695487, + 695519, + 695551, + 695583, + 695647, + 695679, + 695711, + 695743, + 695807, + 695871, + 695967, + 696031, + 696095, + 696159, + 696191, + 696287, + 696319, + 696383, + 696479, + 696511, + 696543, + 696607, + 696735, + 696767, + 696831, + 696863, + 696895, + 696959, + 697023, + 697055, + 697087, + 697119, + 697151, + 697183, + 697215, + 697279, + 697311, + 697343, + 697375, + 697503, + 697535, + 697631, + 697663, + 697695, + 697727, + 697791, + 697823, + 697983, + 698111, + 698143, + 698271, + 698303, + 698367, + 698399, + 698463, + 698495, + 698527, + 698559, + 698591, + 698655, + 698687, + 698751, + 698783, + 698815, + 698879, + 698975, + 699007, + 699135, + 699199, + 699231, + 699327, + 699359, + 699519, + 699551, + 699583, + 699615, + 699647, + 699679, + 699807, + 699839, + 699871, + 699903, + 700095, + 700223, + 700383, + 700479, + 700511, + 700575, + 700639, + 700735, + 700767, + 700799, + 700863, + 700895, + 700959, + 701151, + 701247, + 701311, + 701439, + 701471, + 701503, + 701599, + 701695, + 701727, + 701759, + 701887, + 701919, + 701951, + 701983, + 702015, + 702047, + 702079, + 702111, + 702175, + 702239, + 702303, + 702335, + 702367, + 702463, + 702687, + 702847, + 702879, + 702911, + 702943, + 702975, + 703007, + 703071, + 703103, + 703135, + 703199, + 703263, + 703327, + 703359, + 703423, + 703519, + 703615, + 703647, + 703711, + 703743, + 703775, + 703807, + 703839, + 703871, + 703903, + 703935, + 703967, + 704031, + 704095, + 704191, + 704223, + 704255, + 704287, + 704351, + 704383, + 704415, + 704607, + 704639, + 704735, + 704767, + 704799, + 704831, + 704895, + 704991, + 705023, + 705055, + 705087, + 705119, + 705151, + 705183, + 705215, + 705247, + 705311, + 705471, + 705503, + 705535, + 705567, + 705599, + 705631, + 705663, + 705695, + 705727, + 705759, + 705823, + 706015, + 706047, + 706111, + 706143, + 706175, + 706207, + 706239, + 706271, + 706303, + 706335, + 706367, + 706399, + 706495, + 706527, + 706559, + 706591, + 706623, + 706687, + 706719, + 706751, + 706815, + 706847, + 706911, + 706943, + 706975, + 707039, + 707071, + 707135, + 707231, + 707263, + 707359, + 707455, + 707519, + 707551, + 707583, + 707679, + 707871, + 707903, + 707935, + 707967, + 707999, + 708063, + 708095, + 708351, + 708383, + 708415, + 708479, + 708511, + 708543, + 708607, + 708671, + 708735, + 708799, + 708831, + 708863, + 708959, + 708991, + 709023, + 709087, + 709215, + 709247, + 709279, + 709311, + 709375, + 709407, + 709439, + 709471, + 709535, + 709599, + 709759, + 709791, + 709823, + 709887, + 709919, + 709951, + 709983, + 710015, + 710047, + 710079, + 710111, + 710143, + 710207, + 710239, + 710271, + 710335, + 710399, + 710463, + 710495, + 710559, + 710591, + 710623, + 710719, + 710815, + 710943, + 711039, + 711103, + 711135, + 711199, + 711231, + 711295, + 711327, + 711359, + 711391, + 711423, + 711455, + 711487, + 711551, + 711583, + 711647, + 711711, + 711775, + 711903, + 711935, + 711967, + 711999, + 712031, + 712063, + 712095, + 712127, + 712159, + 712191, + 712223, + 712255, + 712383, + 712447, + 712479, + 712575, + 712607, + 712735, + 712767, + 712927, + 712991, + 713055, + 713087, + 713119, + 713215, + 713247, + 713343, + 713375, + 713407, + 713439, + 713471, + 713535, + 713599, + 713631, + 713695, + 713727, + 713759, + 713791, + 713823, + 713855, + 713919, + 714015, + 714047, + 714079, + 714207, + 714239, + 714303, + 714367, + 714431, + 714495, + 714527, + 714559, + 714591, + 714687, + 714719, + 714751, + 714815, + 714879, + 714943, + 715071, + 715135, + 715199, + 715263, + 715359, + 715391, + 715455, + 715519, + 715551, + 715583, + 715679, + 715711, + 715743, + 715775, + 715903, + 715935, + 716031, + 716063, + 716095, + 716127, + 716159, + 716191, + 716223, + 716287, + 716351, + 716415, + 716543, + 716639, + 716671, + 716703, + 716735, + 716767, + 716863, + 716895, + 716959, + 716991, + 717023, + 717119, + 717151, + 717183, + 717215, + 717311, + 717343, + 717407, + 717471, + 717503, + 717535, + 717631, + 717663, + 717695, + 717759, + 717791, + 717823, + 717887, + 717919, + 718047, + 718079, + 718111, + 718143, + 718175, + 718207, + 718239, + 718367, + 718431, + 718463, + 718527, + 718559, + 718591, + 718623, + 718655, + 718687, + 718719, + 718751, + 718783, + 718815, + 718911, + 719039, + 719167, + 719231, + 719263, + 719295, + 719359, + 719391, + 719423, + 719487, + 719519, + 719551, + 719647, + 719711, + 719743, + 719807, + 719839, + 719871, + 719903, + 719935, + 719967, + 720031, + 720095, + 720159, + 720287, + 720319, + 720351, + 720383, + 720415, + 720447, + 720543, + 720607, + 720639, + 720703, + 720767, + 720799, + 720831, + 720895, + 720927, + 720959, + 720991, + 721087, + 721151, + 721183, + 721279, + 721311, + 721407, + 721439, + 721503, + 721535, + 721567, + 721599, + 721663, + 721695, + 721791, + 721855, + 721887, + 721919, + 721951, + 721983, + 722015, + 722175, + 722207, + 722239, + 722271, + 722303, + 722367, + 722399, + 722431, + 722495, + 722527, + 722591, + 722623, + 722655, + 722687, + 722719, + 722783, + 722847, + 722943, + 722975, + 723071, + 723103, + 723135, + 723167, + 723199, + 723327, + 723359, + 723423, + 723487, + 723551, + 723615, + 723647, + 723743, + 723775, + 724063, + 724095, + 724191, + 724255, + 724287, + 724319, + 724351, + 724383, + 724415, + 724447, + 724511, + 724543, + 724575, + 724607, + 724671, + 724735, + 724863, + 724959, + 724991, + 725055, + 725087, + 725151, + 725247, + 725311, + 725407, + 725599, + 725695, + 725727, + 725791, + 725855, + 725983, + 726111, + 726143, + 726175, + 726207, + 726303, + 726335, + 726367, + 726399, + 726495, + 726527, + 726559, + 726847, + 726879, + 726911, + 726975, + 727007, + 727039, + 727199, + 727231, + 727295, + 727327, + 727359, + 727391, + 727423, + 727455, + 727487, + 727519, + 727551, + 727647, + 727679, + 727743, + 727871, + 727903, + 727935, + 728063, + 728095, + 728127, + 728159, + 728223, + 728287, + 728351, + 728415, + 728511, + 728703, + 728735, + 728831, + 728927, + 728959, + 729023, + 729055, + 729087, + 729119, + 729151, + 729247, + 729279, + 729343, + 729407, + 729471, + 729567, + 729599, + 729695, + 729791, + 729855, + 729919, + 729983, + 730015, + 730079, + 730111, + 730143, + 730335, + 730399, + 730431, + 730463, + 730495, + 730527, + 730591, + 730623, + 730655, + 730687, + 730719, + 730751, + 730783, + 730815, + 730847, + 730911, + 730943, + 731039, + 731135, + 731167, + 731231, + 731263, + 731391, + 731423, + 731455, + 731487, + 731519, + 731615, + 731647, + 731743, + 731807, + 731871, + 731935, + 731967, + 732159, + 732191, + 732223, + 732319, + 732383, + 732511, + 732575, + 732607, + 732639, + 732671, + 732735, + 732799, + 732863, + 732895, + 732927, + 732959, + 732991, + 733023, + 733055, + 733119, + 733247, + 733343, + 733375, + 733407, + 733439, + 733503, + 733567, + 733599, + 733695, + 733727, + 733759, + 733887, + 733951, + 733983, + 734015, + 734047, + 734111, + 734271, + 734303, + 734335, + 734431, + 734463, + 734527, + 734623, + 734687, + 734719, + 734783, + 734815, + 734847, + 734879, + 734943, + 735039, + 735135, + 735199, + 735359, + 735391, + 735455, + 735487, + 735519, + 735583, + 735679, + 735711, + 735775, + 735839, + 735871, + 735903, + 735967, + 735999, + 736031, + 736063, + 736095, + 736127, + 736159, + 736191, + 736223, + 736287, + 736383, + 736415, + 736479, + 736575, + 736639, + 736671, + 736799, + 736831, + 736863, + 736927, + 736959, + 736991, + 737055, + 737183, + 737215, + 737375, + 737503, + 737535, + 737599, + 737663, + 737695, + 737727, + 737791, + 737887, + 737951, + 737983, + 738111, + 738143, + 738335, + 738399, + 738463, + 738495, + 738527, + 738559, + 738591, + 738623, + 738687, + 738751, + 738783, + 738815, + 738879, + 738911, + 739071, + 739167, + 739199, + 739231, + 739263, + 739295, + 739359, + 739455, + 739487, + 739519, + 739551, + 739615, + 739679, + 739711, + 739743, + 739807, + 739839, + 739903, + 739935, + 740063, + 740095, + 740127, + 740191, + 740255, + 740287, + 740383, + 740415, + 740479, + 740511, + 740543, + 740703, + 740735, + 740767, + 740991, + 741151, + 741183, + 741247, + 741343, + 741439, + 741503, + 741567, + 741599, + 741663, + 741727, + 741823, + 741887, + 741919, + 741951, + 741983, + 742079, + 742111, + 742143, + 742207, + 742239, + 742271, + 742335, + 742431, + 742463, + 742527, + 742559, + 742623, + 742687, + 742751, + 742847, + 742879, + 743103, + 743167, + 743199, + 743231, + 743263, + 743295, + 743327, + 743391, + 743423, + 743487, + 743519, + 743551, + 743647, + 743775, + 743807, + 743871, + 743999, + 744031, + 744159, + 744255, + 744287, + 744447, + 744479, + 744511, + 744575, + 744607, + 744639, + 744735, + 744767, + 744799, + 744831, + 744959, + 745023, + 745055, + 745119, + 745151, + 745215, + 745247, + 745279, + 745375, + 745471, + 745567, + 745663, + 745759, + 745791, + 745855, + 745951, + 746047, + 746143, + 746175, + 746207, + 746239, + 746303, + 746335, + 746367, + 746399, + 746431, + 746463, + 746591, + 746751, + 746815, + 746879, + 746911, + 747039, + 747103, + 747135, + 747231, + 747263, + 747295, + 747327, + 747359, + 747391, + 747423, + 747487, + 747583, + 747679, + 747711, + 747839, + 747871, + 747903, + 747999, + 748031, + 748063, + 748127, + 748159, + 748223, + 748255, + 748287, + 748383, + 748415, + 748479, + 748543, + 748607, + 748639, + 748703, + 748735, + 748767, + 748799, + 748831, + 748863, + 748991, + 749055, + 749151, + 749183, + 749215, + 749311, + 749439, + 749503, + 749567, + 749727, + 749791, + 749823, + 749855, + 749887, + 749919, + 749951, + 750111, + 750335, + 750431, + 750463, + 750495, + 750527, + 750559, + 750591, + 750719, + 750847, + 750879, + 750975, + 751039, + 751071, + 751103, + 751135, + 751167, + 751199, + 751231, + 751263, + 751295, + 751327, + 751359, + 751391, + 751423, + 751487, + 751519, + 751583, + 751647, + 751807, + 751967, + 751999, + 752031, + 752095, + 752127, + 752159, + 752223, + 752447, + 752479, + 752511, + 752575, + 752639, + 752671, + 752703, + 752767, + 752863, + 752927, + 752959, + 752991, + 753087, + 753119, + 753183, + 753279, + 753311, + 753375, + 753439, + 753535, + 753599, + 753631, + 753663, + 753695, + 753727, + 753759, + 753823, + 753855, + 753887, + 753919, + 754015, + 754143, + 754175, + 754239, + 754271, + 754303, + 754367, + 754399, + 754431, + 754463, + 754623, + 754655, + 754719, + 754783, + 754847, + 754975, + 755071, + 755103, + 755135, + 755167, + 755391, + 755423, + 755455, + 755519, + 755551, + 755583, + 755615, + 755647, + 755711, + 755743, + 755807, + 755871, + 755903, + 755935, + 756095, + 756127, + 756159, + 756191, + 756223, + 756255, + 756319, + 756383, + 756415, + 756479, + 756543, + 756575, + 756639, + 756703, + 756735, + 756767, + 756799, + 756927, + 757023, + 757055, + 757087, + 757151, + 757183, + 757247, + 757311, + 757375, + 757439, + 757471, + 757567, + 757631, + 757695, + 757759, + 757791, + 757887, + 758015, + 758047, + 758079, + 758111, + 758143, + 758175, + 758271, + 758303, + 758367, + 758399, + 758463, + 758495, + 758527, + 758591, + 758623, + 758687, + 758751, + 758783, + 758815, + 758975, + 759039, + 759071, + 759103, + 759135, + 759167, + 759231, + 759263, + 759327, + 759359, + 759391, + 759519, + 759583, + 759615, + 759743, + 759871, + 759999, + 760063, + 760191, + 760223, + 760255, + 760287, + 760319, + 760351, + 760383, + 760415, + 760447, + 760479, + 760511, + 760575, + 760607, + 760639, + 760703, + 760735, + 760799, + 760927, + 760991, + 761055, + 761087, + 761151, + 761183, + 761279, + 761343, + 761375, + 761407, + 761439, + 761471, + 761503, + 761567, + 761695, + 761759, + 761791, + 761823, + 761855, + 761983, + 762015, + 762047, + 762079, + 762111, + 762143, + 762175, + 762239, + 762335, + 762399, + 762463, + 762559, + 762623, + 762687, + 762783, + 762815, + 762879, + 762911, + 763039, + 763103, + 763135, + 763199, + 763263, + 763423, + 763455, + 763551, + 763583, + 763615, + 763647, + 763679, + 763775, + 763839, + 764031, + 764063, + 764095, + 764319, + 764415, + 764447, + 764543, + 764607, + 764639, + 764703, + 764799, + 764831, + 764863, + 764927, + 764991, + 765055, + 765183, + 765215, + 765247, + 765471, + 765535, + 765599, + 765631, + 765663, + 765727, + 765759, + 765983, + 766047, + 766111, + 766239, + 766271, + 766303, + 766335, + 766399, + 766431, + 766527, + 766559, + 766591, + 766623, + 766783, + 766847, + 766911, + 766943, + 766975, + 767039, + 767071, + 767103, + 767135, + 767199, + 767263, + 767295, + 767327, + 767359, + 767391, + 767455, + 767487, + 767583, + 767679, + 767743, + 767807, + 767839, + 767871, + 767999, + 768287, + 768351, + 768415, + 768543, + 768607, + 768639, + 768671, + 768703, + 768767, + 768799, + 768831, + 768863, + 769023, + 769055, + 769087, + 769151, + 769183, + 769215, + 769279, + 769311, + 769471, + 769535, + 769663, + 769695, + 769727, + 769791, + 769823, + 769855, + 769887, + 770015, + 770079, + 770175, + 770207, + 770239, + 770303, + 770367, + 770431, + 770463, + 770559, + 770655, + 770687, + 770719, + 770815, + 770879, + 770943, + 771007, + 771039, + 771071, + 771103, + 771167, + 771199, + 771231, + 771263, + 771295, + 771327, + 771359, + 771423, + 771647, + 771679, + 771743, + 771775, + 771871, + 771999, + 772063, + 772095, + 772127, + 772287, + 772319, + 772351, + 772447, + 772479, + 772511, + 772543, + 772575, + 772607, + 772671, + 772703, + 772863, + 772927, + 772959, + 772991, + 773055, + 773087, + 773311, + 773343, + 773471, + 773503, + 773535, + 773599, + 773631, + 773695, + 773727, + 773759, + 773791, + 773823, + 773855, + 773919, + 773951, + 773983, + 774047, + 774079, + 774143, + 774207, + 774271, + 774335, + 774367, + 774399, + 774431, + 774559, + 774623, + 774655, + 774719, + 774751, + 774783, + 774815, + 774975, + 775039, + 775071, + 775199, + 775263, + 775295, + 775327, + 775359, + 775391, + 775423, + 775455, + 775519, + 775551, + 775583, + 775679, + 775743, + 775775, + 775839, + 775871, + 775935, + 775999, + 776031, + 776063, + 776095, + 776127, + 776159, + 776223, + 776255, + 776319, + 776415, + 776479, + 776511, + 776543, + 776639, + 776703, + 776735, + 776799, + 776831, + 776863, + 776927, + 777055, + 777119, + 777215, + 777311, + 777343, + 777375, + 777503, + 777535, + 777631, + 777663, + 777727, + 777823, + 777855, + 777983, + 778111, + 778271, + 778303, + 778335, + 778367, + 778399, + 778431, + 778495, + 778559, + 778591, + 778623, + 778655, + 778687, + 778719, + 778783, + 778847, + 778911, + 779007, + 779103, + 779135, + 779167, + 779295, + 779359, + 779423, + 779455, + 779487, + 779583, + 779615, + 779679, + 779711, + 779743, + 779775, + 779903, + 779935, + 779967, + 780063, + 780095, + 780159, + 780287, + 780447, + 780543, + 780575, + 780607, + 780639, + 780799, + 781055, + 781087, + 781215, + 781247, + 781311, + 781343, + 781375, + 781407, + 781471, + 781663, + 781695, + 781727, + 781791, + 781823, + 781887, + 781983, + 782015, + 782047, + 782079, + 782143, + 782207, + 782239, + 782271, + 782335, + 782367, + 782399, + 782431, + 782495, + 782687, + 782751, + 782783, + 782815, + 782847, + 782879, + 783103, + 783135, + 783167, + 783231, + 783263, + 783295, + 783327, + 783359, + 783423, + 783487, + 783519, + 783615, + 783647, + 783743, + 783775, + 783807, + 783839, + 783903, + 783935, + 783967, + 784031, + 784095, + 784255, + 784351, + 784383, + 784447, + 784479, + 784543, + 784607, + 784639, + 784703, + 784735, + 784799, + 784831, + 784863, + 784895, + 784927, + 784959, + 784991, + 785087, + 785119, + 785247, + 785311, + 785343, + 785439, + 785471, + 785663, + 785695, + 785759, + 785919, + 785951, + 785983, + 786079, + 786239, + 786303, + 786335, + 786591, + 786655, + 786687, + 786719, + 786783, + 786815, + 786911, + 786943, + 787007, + 787135, + 787231, + 787295, + 787327, + 787359, + 787423, + 787455, + 787711, + 787743, + 787839, + 787903, + 787999, + 788031, + 788127, + 788159, + 788191, + 788319, + 788351, + 788447, + 788479, + 788511, + 788575, + 788607, + 788671, + 788703, + 788831, + 789055, + 789119, + 789151, + 789247, + 789343, + 789375, + 789471, + 789503, + 789535, + 789695, + 789759, + 789855, + 789887, + 790015, + 790047, + 790079, + 790143, + 790175, + 790207, + 790271, + 790335, + 790399, + 790431, + 790463, + 790559, + 790591, + 790623, + 790655, + 790719, + 790751, + 790815, + 790879, + 790911, + 790943, + 791007, + 791039, + 791135, + 791167, + 791199, + 791231, + 791263, + 791359, + 791423, + 791615, + 791647, + 791679, + 791711, + 791743, + 791775, + 791839, + 791935, + 791967, + 792095, + 792223, + 792255, + 792319, + 792351, + 792383, + 792415, + 792479, + 792575, + 792671, + 792767, + 792799, + 792831, + 792863, + 792959, + 793055, + 793215, + 793279, + 793311, + 793343, + 793407, + 793503, + 793567, + 793663, + 793695, + 793727, + 793759, + 793791, + 794047, + 794079, + 794111, + 794143, + 794175, + 794239, + 794271, + 794367, + 794431, + 794495, + 794655, + 794687, + 794719, + 794783, + 794815, + 794943, + 794975, + 795071, + 795103, + 795167, + 795295, + 795327, + 795359, + 795487, + 795519, + 795551, + 795711, + 795839, + 795999, + 796031, + 796063, + 796191, + 796223, + 796255, + 796287, + 796383, + 796447, + 796479, + 796543, + 796575, + 796607, + 796703, + 796767, + 796831, + 796863, + 796927, + 797023, + 797055, + 797087, + 797119, + 797247, + 797279, + 797375, + 797471, + 797503, + 797535, + 797567, + 797631, + 797663, + 797695, + 797727, + 797759, + 797791, + 797887, + 797919, + 797951, + 797983, + 798047, + 798175, + 798207, + 798239, + 798271, + 798367, + 798399, + 798463, + 798559, + 798591, + 798655, + 798719, + 798815, + 798879, + 799071, + 799199, + 799263, + 799295, + 799327, + 799359, + 799455, + 799583, + 799711, + 799807, + 799871, + 799967, + 799999, + 800031, + 800159, + 800223, + 800255, + 800319, + 800351, + 800415, + 800479, + 800511, + 800543, + 800575, + 800639, + 800863, + 800927, + 800959, + 801023, + 801055, + 801119, + 801151, + 801183, + 801343, + 801407, + 801471, + 801503, + 801567, + 801599, + 801695, + 801727, + 801791, + 801855, + 801919, + 801951, + 801983, + 802015, + 802143, + 802175, + 802271, + 802303, + 802399, + 802463, + 802495, + 802591, + 802623, + 802719, + 802751, + 802783, + 802943, + 803071, + 803103, + 803135, + 803167, + 803199, + 803295, + 803391, + 803519, + 803615, + 803647, + 803679, + 803711, + 803807, + 803871, + 803999, + 804031, + 804095, + 804127, + 804223, + 804255, + 804319, + 804383, + 804415, + 804575, + 804607, + 804671, + 804863, + 804927, + 804991, + 805023, + 805119, + 805215, + 805247, + 805343, + 805439, + 805471, + 805535, + 805567, + 805695, + 805727, + 805791, + 805823, + 805855, + 806047, + 806079, + 806143, + 806239, + 806271, + 806335, + 806367, + 806399, + 806463, + 806495, + 806687, + 806783, + 806815, + 807007, + 807039, + 807199, + 807231, + 807295, + 807327, + 807583, + 807615, + 807679, + 807743, + 807775, + 807871, + 807967, + 807999, + 808031, + 808063, + 808095, + 808191, + 808287, + 808319, + 808479, + 808607, + 808703, + 808735, + 808767, + 808895, + 808991, + 809023, + 809151, + 809279, + 809407, + 809439, + 809471, + 809535, + 809631, + 809663, + 809727, + 809823, + 809919, + 809951, + 809983, + 810015, + 810047, + 810079, + 810111, + 810143, + 810303, + 810367, + 810399, + 810463, + 810495, + 810623, + 810655, + 810687, + 810783, + 810847, + 810943, + 811039, + 811135, + 811167, + 811263, + 811327, + 811359, + 811423, + 811519, + 811551, + 811583, + 811615, + 811647, + 811679, + 811711, + 811807, + 811839, + 811871, + 811903, + 811935, + 811967, + 811999, + 812031, + 812095, + 812127, + 812159, + 812223, + 812319, + 812351, + 812383, + 812543, + 812639, + 812671, + 812703, + 812735, + 812895, + 812959, + 812991, + 813023, + 813151, + 813183, + 813215, + 813375, + 813407, + 813439, + 813503, + 813663, + 813727, + 813855, + 813887, + 814015, + 814079, + 814143, + 814271, + 814431, + 814591, + 814623, + 814687, + 814719, + 814751, + 814815, + 814847, + 814911, + 814943, + 814975, + 815103, + 815167, + 815231, + 815519, + 815551, + 815583, + 815615, + 815743, + 815775, + 815839, + 815871, + 815967, + 815999, + 816095, + 816127, + 816159, + 816191, + 816223, + 816255, + 816319, + 816447, + 816511, + 816543, + 816575, + 816607, + 816639, + 816735, + 816799, + 816863, + 816927, + 816991, + 817023, + 817055, + 817087, + 817183, + 817247, + 817311, + 817407, + 817439, + 817471, + 817503, + 817535, + 817567, + 817599, + 817727, + 817759, + 817823, + 817855, + 817887, + 817983, + 818047, + 818111, + 818143, + 818207, + 818271, + 818367, + 818399, + 818495, + 818527, + 818591, + 818623, + 818655, + 818719, + 818751, + 818783, + 818815, + 818911, + 818943, + 818975, + 819007, + 819039, + 819199, + 819231, + 819327, + 819391, + 819583, + 819679, + 819743, + 819871, + 819903, + 820127, + 820191, + 820223, + 820255, + 820287, + 820351, + 820415, + 820479, + 820511, + 820543, + 820639, + 820671, + 820799, + 820863, + 821023, + 821055, + 821087, + 821151, + 821183, + 821311, + 821343, + 821407, + 821535, + 821567, + 821631, + 821663, + 821727, + 821823, + 821855, + 821887, + 821919, + 822175, + 822207, + 822239, + 822271, + 822367, + 822399, + 822431, + 822463, + 822591, + 822751, + 822911, + 823199, + 823263, + 823391, + 823423, + 823519, + 823551, + 823615, + 823743, + 823807, + 823839, + 823935, + 824095, + 824127, + 824159, + 824191, + 824287, + 824383, + 824479, + 824511, + 824543, + 824575, + 824639, + 824671, + 824735, + 824799, + 824863, + 824927, + 824959, + 824991, + 825023, + 825151, + 825247, + 825343, + 825375, + 825471, + 825535, + 825567, + 825599, + 825631, + 825663, + 825695, + 825727, + 825823, + 825951, + 826143, + 826175, + 826207, + 826239, + 826271, + 826335, + 826431, + 826463, + 826527, + 826559, + 826687, + 826719, + 826751, + 826783, + 826815, + 826879, + 826911, + 826975, + 827007, + 827039, + 827071, + 827135, + 827167, + 827199, + 827263, + 827327, + 827487, + 827647, + 827679, + 827711, + 827743, + 827775, + 827871, + 827999, + 828031, + 828127, + 828159, + 828255, + 828287, + 828351, + 828511, + 828543, + 828575, + 828607, + 828671, + 828703, + 828767, + 828799, + 828863, + 828927, + 828959, + 829087, + 829151, + 829183, + 829407, + 829471, + 829503, + 829535, + 829631, + 829727, + 829759, + 829823, + 829855, + 829887, + 829983, + 830207, + 830239, + 830271, + 830367, + 830399, + 830431, + 830463, + 830527, + 830591, + 830623, + 830655, + 830687, + 830815, + 830847, + 830911, + 831039, + 831103, + 831263, + 831359, + 831391, + 831487, + 831583, + 831711, + 831871, + 831935, + 832031, + 832063, + 832159, + 832191, + 832223, + 832319, + 832351, + 832383, + 832415, + 832511, + 832639, + 832671, + 832767, + 832863, + 832927, + 832991, + 833023, + 833055, + 833247, + 833311, + 833375, + 833503, + 833631, + 833663, + 833727, + 833791, + 833855, + 834047, + 834079, + 834175, + 834271, + 834335, + 834431, + 834463, + 834591, + 834623, + 834847, + 834879, + 834911, + 834943, + 834975, + 835007, + 835071, + 835103, + 835231, + 835295, + 835327, + 835391, + 835487, + 835583, + 835743, + 836063, + 836095, + 836127, + 836255, + 836287, + 836447, + 836543, + 836575, + 836639, + 836767, + 836895, + 836959, + 836991, + 837055, + 837119, + 837183, + 837215, + 837247, + 837279, + 837343, + 837375, + 837631, + 837759, + 837887, + 838079, + 838143, + 838207, + 838239, + 838271, + 838399, + 838495, + 838527, + 838591, + 838623, + 838655, + 838943, + 838975, + 839007, + 839039, + 839135, + 839231, + 839295, + 839359, + 839615, + 839647, + 839743, + 839807, + 839871, + 839903, + 839967, + 840063, + 840159, + 840287, + 840319, + 840351, + 840447, + 840543, + 840607, + 840671, + 840703, + 840767, + 840799, + 840863, + 840991, + 841055, + 841087, + 841183, + 841215, + 841247, + 841279, + 841439, + 841471, + 841567, + 841599, + 841663, + 841695, + 841727, + 841759, + 841791, + 841951, + 841983, + 842047, + 842111, + 842175, + 842271, + 842303, + 842335, + 842431, + 842495, + 842527, + 842559, + 842719, + 842847, + 842943, + 843039, + 843167, + 843231, + 843295, + 843327, + 843359, + 843391, + 843423, + 843455, + 843487, + 843519, + 843551, + 843615, + 843711, + 843743, + 843775, + 843807, + 843871, + 843999, + 844063, + 844095, + 844255, + 844351, + 844383, + 844415, + 844607, + 844671, + 844767, + 844799, + 844831, + 844895, + 845087, + 845151, + 845183, + 845215, + 845375, + 845535, + 845567, + 845599, + 845791, + 845855, + 845919, + 845983, + 846047, + 846079, + 846111, + 846143, + 846271, + 846303, + 846335, + 846367, + 846495, + 846591, + 846687, + 846719, + 846783, + 846815, + 846847, + 846911, + 846943, + 847007, + 847199, + 847231, + 847263, + 847327, + 847391, + 847423, + 847487, + 847519, + 847551, + 847679, + 847711, + 847743, + 847807, + 847871, + 847903, + 847935, + 847967, + 847999, + 848159, + 848191, + 848255, + 848415, + 848447, + 848511, + 848575, + 848607, + 848671, + 848703, + 848735, + 848863, + 848991, + 849183, + 849311, + 849343, + 849535, + 849567, + 849695, + 849791, + 849855, + 849983, + 850015, + 850047, + 850111, + 850143, + 850207, + 850271, + 850303, + 850367, + 850463, + 850559, + 850591, + 850623, + 850687, + 850751, + 850783, + 850847, + 850879, + 851007, + 851103, + 851263, + 851295, + 851359, + 851615, + 851679, + 851807, + 851871, + 851903, + 851935, + 851999, + 852031, + 852095, + 852159, + 852191, + 852223, + 852255, + 852287, + 852319, + 852351, + 852415, + 852447, + 852575, + 852607, + 852639, + 852703, + 852735, + 852895, + 852991, + 853087, + 853151, + 853247, + 853279, + 853663, + 853695, + 853759, + 853823, + 853855, + 853887, + 853951, + 853983, + 854015, + 854271, + 854303, + 854335, + 854399, + 854431, + 854527, + 854623, + 854751, + 854815, + 854911, + 854943, + 854975, + 855071, + 855103, + 855135, + 855487, + 855551, + 855583, + 855679, + 855903, + 855935, + 855999, + 856031, + 856063, + 856223, + 856255, + 856319, + 856351, + 856383, + 856447, + 856479, + 856543, + 856639, + 856703, + 856735, + 856863, + 856927, + 857055, + 857087, + 857119, + 857183, + 857279, + 857343, + 857375, + 857407, + 857471, + 857567, + 857599, + 857631, + 857695, + 857727, + 857823, + 857919, + 857983, + 858015, + 858079, + 858111, + 858143, + 858271, + 858335, + 858367, + 858399, + 858431, + 858527, + 858655, + 858687, + 858815, + 858975, + 859007, + 859071, + 859135, + 859167, + 859263, + 859295, + 859359, + 859391, + 859487, + 859519, + 859583, + 859647, + 859807, + 859871, + 860031, + 860063, + 860095, + 860127, + 860159, + 860223, + 860287, + 860319, + 860351, + 860415, + 860511, + 860607, + 860639, + 860703, + 860895, + 860959, + 860991, + 861023, + 861055, + 861087, + 861183, + 861311, + 861343, + 861407, + 861471, + 861535, + 861567, + 861631, + 861727, + 861759, + 861823, + 861887, + 861919, + 861983, + 862079, + 862143, + 862239, + 862271, + 862367, + 862527, + 862623, + 862655, + 862751, + 862815, + 862879, + 863103, + 863135, + 863199, + 863263, + 863359, + 863423, + 863583, + 863615, + 863647, + 863743, + 863807, + 863839, + 863871, + 863999, + 864095, + 864127, + 864159, + 864223, + 864255, + 864287, + 864351, + 864415, + 864479, + 864575, + 864607, + 864735, + 864831, + 864895, + 864927, + 864991, + 865087, + 865151, + 865215, + 865311, + 865343, + 865407, + 865439, + 865503, + 865535, + 865599, + 865631, + 865663, + 865727, + 865759, + 865791, + 865951, + 866175, + 866239, + 866335, + 866367, + 866399, + 866463, + 866495, + 866719, + 866751, + 866783, + 866815, + 866847, + 866879, + 866943, + 867039, + 867071, + 867103, + 867135, + 867199, + 867263, + 867359, + 867487, + 867551, + 867679, + 867807, + 867935, + 867967, + 868031, + 868095, + 868127, + 868255, + 868383, + 868543, + 868575, + 868607, + 868671, + 868767, + 868799, + 868927, + 868959, + 869023, + 869055, + 869151, + 869247, + 869279, + 869343, + 869407, + 869439, + 869471, + 869503, + 869695, + 869727, + 869791, + 869823, + 869855, + 869919, + 869951, + 870111, + 870175, + 870271, + 870431, + 870463, + 870495, + 870623, + 870687, + 870751, + 870815, + 870879, + 870911, + 870943, + 871007, + 871039, + 871135, + 871167, + 871295, + 871327, + 871359, + 871391, + 871423, + 871583, + 871615, + 871679, + 871711, + 871775, + 871871, + 871967, + 872063, + 872159, + 872223, + 872351, + 872383, + 872447, + 872607, + 872639, + 872703, + 872767, + 872863, + 872895, + 872927, + 873023, + 873055, + 873087, + 873151, + 873183, + 873215, + 873247, + 873279, + 873311, + 873343, + 873407, + 873439, + 873535, + 873663, + 873695, + 873823, + 873855, + 873887, + 873919, + 873951, + 874015, + 874047, + 874143, + 874207, + 874239, + 874335, + 874431, + 874495, + 874527, + 874591, + 874623, + 874719, + 874751, + 874783, + 874847, + 874879, + 874943, + 874975, + 875039, + 875071, + 875103, + 875199, + 875295, + 875359, + 875391, + 875423, + 875455, + 875487, + 875519, + 875551, + 875583, + 875615, + 875647, + 875743, + 875775, + 875871, + 875903, + 875935, + 875967, + 875999, + 876031, + 876159, + 876191, + 876223, + 876255, + 876351, + 876447, + 876511, + 876639, + 876671, + 876799, + 876863, + 876927, + 877055, + 877119, + 877279, + 877311, + 877375, + 877407, + 877567, + 877599, + 877663, + 877695, + 877823, + 877887, + 877919, + 877983, + 878047, + 878079, + 878111, + 878143, + 878207, + 878271, + 878303, + 878463, + 878495, + 878527, + 878559, + 878623, + 878655, + 878719, + 878751, + 878815, + 879007, + 879071, + 879167, + 879199, + 879231, + 879295, + 879391, + 879455, + 879583, + 879615, + 879647, + 879743, + 879967, + 879999, + 880063, + 880095, + 880127, + 880159, + 880447, + 880575, + 880607, + 880639, + 880671, + 880703, + 880735, + 880799, + 880831, + 880863, + 880895, + 881023, + 881087, + 881119, + 881215, + 881247, + 881311, + 881375, + 881471, + 881535, + 881567, + 881631, + 881823, + 881855, + 881887, + 881951, + 882047, + 882143, + 882175, + 882239, + 882463, + 882527, + 882559, + 882719, + 882815, + 882847, + 882879, + 882911, + 882975, + 883007, + 883039, + 883103, + 883167, + 883327, + 883359, + 883423, + 883455, + 883487, + 883519, + 883615, + 883679, + 883743, + 883839, + 883999, + 884095, + 884127, + 884159, + 884191, + 884223, + 884479, + 884511, + 884639, + 884671, + 884703, + 884767, + 884799, + 884831, + 884895, + 884959, + 885151, + 885279, + 885407, + 885503, + 885535, + 885631, + 885663, + 885759, + 885919, + 885951, + 886047, + 886079, + 886143, + 886239, + 886367, + 886495, + 886527, + 886591, + 886655, + 886719, + 886815, + 886879, + 886943, + 887039, + 887071, + 887167, + 887199, + 887327, + 887359, + 887391, + 887487, + 887519, + 887615, + 887743, + 887871, + 887935, + 887999, + 888063, + 888095, + 888127, + 888223, + 888287, + 888319, + 888383, + 888447, + 888479, + 888607, + 888671, + 888767, + 888799, + 888831, + 888927, + 889023, + 889087, + 889119, + 889151, + 889183, + 889247, + 889279, + 889311, + 889343, + 889375, + 889407, + 889439, + 889535, + 889567, + 889631, + 889663, + 889727, + 889791, + 889823, + 889887, + 890015, + 890271, + 890335, + 890431, + 890623, + 890687, + 890719, + 890751, + 890815, + 890879, + 890975, + 891039, + 891135, + 891167, + 891199, + 891263, + 891295, + 891327, + 891359, + 891455, + 891583, + 891615, + 891647, + 891743, + 891871, + 891903, + 891935, + 891967, + 891999, + 892095, + 892127, + 892255, + 892287, + 892319, + 892351, + 892479, + 892511, + 892575, + 892607, + 892639, + 892671, + 892703, + 892799, + 892831, + 892895, + 892927, + 893119, + 893215, + 893247, + 893311, + 893343, + 893407, + 893503, + 893599, + 893695, + 893727, + 893823, + 893855, + 893887, + 893983, + 894015, + 894079, + 894143, + 894175, + 894303, + 894335, + 894367, + 894527, + 894559, + 894591, + 894623, + 894719, + 894751, + 894783, + 894911, + 894943, + 895039, + 895071, + 895167, + 895231, + 895263, + 895327, + 895359, + 895391, + 895423, + 895455, + 895487, + 895519, + 895647, + 895871, + 895935, + 895999, + 896063, + 896127, + 896223, + 896255, + 896287, + 896447, + 896479, + 896575, + 896639, + 896671, + 896703, + 896735, + 896831, + 896927, + 897183, + 897247, + 897503, + 897599, + 897631, + 897695, + 897919, + 898079, + 898111, + 898175, + 898207, + 898239, + 898271, + 898303, + 898367, + 898431, + 898463, + 898559, + 898591, + 898655, + 898751, + 898783, + 898943, + 899135, + 899167, + 899263, + 899423, + 899583, + 899615, + 899647, + 899679, + 899839, + 899871, + 899903, + 900063, + 900191, + 900287, + 900319, + 900383, + 900415, + 900447, + 900479, + 900607, + 900767, + 900863, + 900927, + 901151, + 901183, + 901279, + 901311, + 901567, + 901599, + 901791, + 901823, + 901919, + 902111, + 902175, + 902303, + 902335, + 902399, + 902431, + 902591, + 902751, + 902847, + 902879, + 902911, + 902943, + 903007, + 903071, + 903167, + 903199, + 903359, + 903423, + 903583, + 903615, + 903647, + 903679, + 903711, + 903871, + 903903, + 903967, + 904095, + 904191, + 904255, + 904415, + 904511, + 904543, + 904927, + 904959, + 905023, + 905183, + 905247, + 905471, + 905535, + 905599, + 906079, + 906143, + 906431, + 906463, + 906591, + 906687, + 906719, + 906751, + 906783, + 906815, + 907167, + 907231, + 907295, + 907391, + 907455, + 907519, + 907679, + 907711, + 907871, + 908031, + 908063, + 908095, + 908191, + 908639, + 908703, + 908895, + 908927, + 908991, + 909055, + 909087, + 909119, + 909215, + 909247, + 909471, + 909567, + 909823, + 909855, + 909887, + 909983, + 910175, + 910239, + 910271, + 910367, + 910719, + 910751, + 910783, + 910879, + 910911, + 911039, + 911103, + 911231, + 911647, + 911743, + 911807, + 911935, + 911967, + 912031, + 912127, + 912191, + 912223, + 912287, + 912319, + 912351, + 912575, + 912767, + 912799, + 912863, + 912927, + 913055, + 913311, + 913407, + 913439, + 913471, + 913535, + 913663, + 913695, + 913727, + 913759, + 913791, + 913887, + 913951, + 914239, + 914271, + 914335, + 914527, + 914591, + 914719, + 914815, + 914847, + 914911, + 915039, + 915071, + 915103, + 915231, + 915295, + 915391, + 915455, + 915487, + 915519, + 915583, + 915711, + 915743, + 915807, + 915967, + 916095, + 916127, + 916255, + 916319, + 916351, + 916383, + 916447, + 916639, + 916895, + 916927, + 916959, + 916991, + 917087, + 917151, + 917183, + 917247, + 917279, + 917311, + 917375, + 917407, + 917471, + 917663, + 917823, + 917855, + 917951, + 918143, + 918175, + 918335, + 918399, + 918559, + 918591, + 918847, + 918879, + 918975, + 919039, + 919199, + 919231, + 919295, + 919327, + 919359, + 919391, + 919455, + 919487, + 919551, + 919679, + 919711, + 919743, + 919807, + 919967, + 920095, + 920127, + 920255, + 920383, + 920415, + 920639, + 920671, + 920735, + 920831, + 920863, + 920927, + 920959, + 921023, + 921055, + 921087, + 921183, + 921247, + 921311, + 921439, + 921695, + 921919, + 922111, + 922143, + 922207, + 922239, + 922271, + 922335, + 922367, + 922815, + 922911, + 923039, + 923167, + 923263, + 923487, + 923615, + 923679, + 923807, + 923903, + 923935, + 923999, + 924031, + 924095, + 924127, + 924159, + 924191, + 924223, + 924383, + 924447, + 924479, + 924511, + 924671, + 924863, + 924927, + 925215, + 925311, + 925375, + 925407, + 925439, + 925471, + 925503, + 925631, + 925663, + 925759, + 925823, + 926047, + 926079, + 926111, + 926207, + 926399, + 926495, + 926623, + 926687, + 926719, + 926751, + 926975, + 927103, + 927167, + 927263, + 927423, + 927455, + 927519, + 927615, + 927679, + 927711, + 927807, + 927871, + 927903, + 927999, + 928159, + 928287, + 928383, + 928447, + 928479, + 928575, + 928735, + 928767, + 928831, + 928991, + 929023, + 929087, + 929247, + 929471, + 929599, + 929759, + 929823, + 930207, + 930303, + 930367, + 930431, + 930463, + 930495, + 930527, + 930623, + 930815, + 930879, + 931007, + 931039, + 931199, + 931327, + 931615, + 931711, + 931871, + 931999, + 932255, + 932287, + 932415, + 932447, + 932479, + 932639, + 932671, + 932895, + 932927, + 932959, + 933055, + 933087, + 933215, + 933311, + 933471, + 933631, + 933727, + 933823, + 933887, + 933951, + 933983, + 934047, + 934207, + 934271, + 934367, + 934399, + 934655, + 934719, + 934751, + 934879, + 934911, + 935039, + 935135, + 935263, + 935391, + 935455, + 935487, + 935743, + 935807, + 935839, + 935903, + 936031, + 936191, + 936223, + 936255, + 936351, + 936383, + 936511, + 936735, + 936799, + 936895, + 936927, + 937247, + 937311, + 937599, + 937663, + 937727, + 938111, + 938175, + 938207, + 938239, + 938303, + 938335, + 938367, + 938463, + 938591, + 938751, + 938911, + 939135, + 939167, + 939231, + 939327, + 939359, + 939519, + 939551, + 939615, + 939679, + 940191, + 940223, + 940255, + 940383, + 940415, + 940543, + 940607, + 940735, + 940927, + 941247, + 941311, + 941343, + 941375, + 941471, + 941567, + 941599, + 941727, + 941887, + 941983, + 942303, + 942335, + 942367, + 942463, + 942495, + 942719, + 942783, + 942975, + 943071, + 943167, + 943199, + 943423, + 943679, + 943711, + 943775, + 943807, + 943935, + 944031, + 944095, + 944159, + 944191, + 944223, + 944447, + 944479, + 944607, + 944703, + 944767, + 944831, + 945247, + 945407, + 945439, + 945503, + 945535, + 945567, + 945599, + 945663, + 945695, + 945759, + 945791, + 945823, + 945855, + 945951, + 946015, + 946175, + 946207, + 946271, + 946431, + 946495, + 946527, + 946751, + 947231, + 947615, + 947647, + 947679, + 947711, + 947775, + 947839, + 947967, + 948223, + 948543, + 948607, + 948671, + 948703, + 948735, + 948767, + 948863, + 948991, + 949151, + 949247, + 949375, + 949439, + 949503, + 949759, + 949951, + 950015, + 950079, + 950143, + 950239, + 950399, + 950559, + 950687, + 950719, + 950847, + 950975, + 951007, + 951135, + 951295, + 951327, + 951583, + 951615, + 951743, + 951871, + 951903, + 951935, + 951999, + 952191, + 952255, + 952415, + 952447, + 952607, + 952703, + 952735, + 952831, + 952991, + 953023, + 953247, + 953279, + 953375, + 953439, + 953535, + 953759, + 953791, + 953823, + 953983, + 954079, + 954111, + 954239, + 954335, + 954559, + 954623, + 954687, + 954719, + 954847, + 954879, + 954911, + 955199, + 955327, + 955391, + 955583, + 955711, + 955807, + 955935, + 955967, + 956031, + 956127, + 956159, + 956223, + 956351, + 956447, + 956543, + 956575, + 956863, + 956927, + 957119, + 957247, + 957311, + 957375, + 957503, + 957567, + 957663, + 957759, + 957951, + 958015, + 958079, + 958143, + 958207, + 958239, + 958335, + 958431, + 958495, + 958591, + 958783, + 958943, + 959103, + 959199, + 959295, + 959711, + 959743, + 959775, + 959807, + 960191, + 960319, + 960415, + 960511, + 960767, + 960831, + 961023, + 961439, + 961567, + 961695, + 961791, + 961823, + 961951, + 961983, + 962079, + 962239, + 962367, + 962719, + 962751, + 962879, + 962975, + 963263, + 963359, + 963423, + 963487, + 963647, + 963679, + 963871, + 964159, + 964543, + 964639, + 964767, + 964959, + 964991, + 965023, + 965087, + 965279, + 965311, + 965343, + 965375, + 965471, + 965727, + 965759, + 965855, + 965983, + 966047, + 966111, + 966271, + 966367, + 966431, + 966463, + 966495, + 966783, + 966815, + 966911, + 966943, + 967295, + 967551, + 967743, + 967807, + 967871, + 967999, + 968031, + 968063, + 968095, + 968127, + 968159, + 968223, + 968255, + 968287, + 968383, + 968447, + 968671, + 968703, + 968831, + 968927, + 968959, + 968991, + 969023, + 969151, + 969215, + 969247, + 969279, + 969343, + 969567, + 969631, + 969727, + 969887, + 970047, + 970111, + 970143, + 970175, + 970495, + 970527, + 970623, + 970751, + 970783, + 970975, + 971007, + 971071, + 971103, + 971263, + 971295, + 971583, + 971775, + 971999, + 972095, + 972415, + 972639, + 972703, + 972831, + 972895, + 973311, + 973375, + 973439, + 973471, + 973503, + 973663, + 973759, + 973855, + 973951, + 974015, + 974047, + 974175, + 974239, + 974367, + 974495, + 974783, + 974911, + 975135, + 975231, + 975391, + 975647, + 975711, + 975743, + 975839, + 976063, + 976127, + 976319, + 976447, + 976479, + 976607, + 976703, + 976735, + 976767, + 976959, + 976991, + 977023, + 977279, + 977375, + 977407, + 977439, + 977599, + 977727, + 977887, + 977983, + 978111, + 978175, + 978207, + 978271, + 978335, + 978495, + 978527, + 978559, + 978623, + 978975, + 979039, + 979199, + 979423, + 979807, + 979967, + 980063, + 980223, + 980383, + 980639, + 980735, + 980927, + 981023, + 981215, + 981311, + 981503, + 981631, + 981823, + 981919, + 981951, + 982207, + 982271, + 982591, + 982623, + 982719, + 982751, + 982911, + 982943, + 983135, + 983775, + 983903, + 983999, + 984287, + 984351, + 984607, + 984703, + 984831, + 985151, + 985247, + 985375, + 985471, + 985535, + 985599, + 985695, + 985759, + 985823, + 985855, + 986047, + 986367, + 986463, + 986495, + 986527, + 986655, + 986879, + 986975, + 987103, + 987263, + 987327, + 987487, + 987615, + 987711, + 987743, + 987775, + 987999, + 988031, + 988191, + 988287, + 988319, + 988351, + 988415, + 988479, + 988767, + 988863, + 988895, + 989055, + 989087, + 989183, + 989215, + 989343, + 989407, + 989439, + 989471, + 989535, + 989663, + 990079, + 990143, + 990207, + 990239, + 990367, + 990527, + 990559, + 990623, + 990719, + 990815, + 990847, + 990975, + 991071, + 991199, + 991231, + 991519, + 991551, + 991583, + 991775, + 991871, + 991967, + 992159, + 992255, + 992479, + 992639, + 992703, + 993087, + 993151, + 993279, + 993343, + 993471, + 993567, + 993663, + 993951, + 994079, + 994143, + 994527, + 994591, + 994655, + 994815, + 994943, + 995071, + 995103, + 995167, + 995263, + 995391, + 995519, + 995615, + 995679, + 995711, + 995839, + 995903, + 996063, + 996159, + 996351, + 996575, + 996607, + 996671, + 996767, + 996831, + 996863, + 996927, + 997151, + 997215, + 997599, + 997951, + 997983, + 998143, + 998175, + 998207, + 998303, + 998367, + 998463, + 998559, + 998591, + 998623, + 998655, + 998847, + 998911, + 999071, + 999167, + 999391, + 999423, + 999519, + 999775, + 999935, + 1000351, + 1000383, + 1000415, + 1000447, + 1000511, + 1000703, + 1000735, + 1000863, + 1000895, + 1001119, + 1001151, + 1001407, + 1001503, + 1001535, + 1001567, + 1001631, + 1001727, + 1001759, + 1002079, + 1002111, + 1002271, + 1002303, + 1002335, + 1002463, + 1002495, + 1002527, + 1002655, + 1002847, + 1002879, + 1002911, + 1003167, + 1003615, + 1003839, + 1003935, + 1003967, + 1004031, + 1004159, + 1004191, + 1004351, + 1004671, + 1004735, + 1004767, + 1004927, + 1005055, + 1005503, + 1005535, + 1005951, + 1006271, + 1006335, + 1006591, + 1006655, + 1006687, + 1006911, + 1007071, + 1007135, + 1007231, + 1007551, + 1007647, + 1007679, + 1007711, + 1007743, + 1007999, + 1008063, + 1008191, + 1008255, + 1008287, + 1008319, + 1008383, + 1008415, + 1008479, + 1008575, + 1008703, + 1008735, + 1008831, + 1008863, + 1008895, + 1008959, + 1009343, + 1009407, + 1009567, + 1009663, + 1009695, + 1009887, + 1009951, + 1010015, + 1010079, + 1010143, + 1010271, + 1010431, + 1010463, + 1010527, + 1010591, + 1010655, + 1010719, + 1010815, + 1010847, + 1010911, + 1011103, + 1011231, + 1011327, + 1011487, + 1011615, + 1011647, + 1011839, + 1011967, + 1012063, + 1012223, + 1012543, + 1012575, + 1012607, + 1012639, + 1012799, + 1013151, + 1013183, + 1013439, + 1013791, + 1013951, + 1013983, + 1014015, + 1014143, + 1014175, + 1014207, + 1014271, + 1014431, + 1014591, + 1014623, + 1014751, + 1014847, + 1014879, + 1014943, + 1015295, + 1015359, + 1015455, + 1015583, + 1015615, + 1015647, + 1015871, + 1015903, + 1015935, + 1015967, + 1015999, + 1016383, + 1016671, + 1016703, + 1016735, + 1016767, + 1016831, + 1017023, + 1017087, + 1017183, + 1017407, + 1017439, + 1017535, + 1017567, + 1017663, + 1017791, + 1017823, + 1017855, + 1018207, + 1018303, + 1018527, + 1018719, + 1018751, + 1018815, + 1018847, + 1019007, + 1019263, + 1019359, + 1019423, + 1019583, + 1019647, + 1019743, + 1019839, + 1019999, + 1020031, + 1020063, + 1020415, + 1020479, + 1020575, + 1020639, + 1020671, + 1020703, + 1020735, + 1020799, + 1020991, + 1021055, + 1021087, + 1021119, + 1021151, + 1021215, + 1021311, + 1021503, + 1021695, + 1021759, + 1021823, + 1021887, + 1021983, + 1022111, + 1022143, + 1022175, + 1022687, + 1022783, + 1023007, + 1023039, + 1023071, + 1023295, + 1023487, + 1023871, + 1023967, + 1024159, + 1024191, + 1024223, + 1024383, + 1024415, + 1024511, + 1024735, + 1024959, + 1025055, + 1025567, + 1025695, + 1025887, + 1026015, + 1026175, + 1026303, + 1026335, + 1026431, + 1026847, + 1026911, + 1026943, + 1027039, + 1027199, + 1027231, + 1027423, + 1027647, + 1027839, + 1027871, + 1028223, + 1028383, + 1028415, + 1028447, + 1028607, + 1029055, + 1029119, + 1029215, + 1029567, + 1029855, + 1030015, + 1030047, + 1030111, + 1030271, + 1030495, + 1030655, + 1030719, + 1030751, + 1030847, + 1030975, + 1031007, + 1031071, + 1031167, + 1031263, + 1031391, + 1031519, + 1031551, + 1031583, + 1031839, + 1031967, + 1032031, + 1032159, + 1032511, + 1032671, + 1032703, + 1032799, + 1032991, + 1033151, + 1033183, + 1033503, + 1033695, + 1033727, + 1033759, + 1033791, + 1033919, + 1033983, + 1034175, + 1034207, + 1034303, + 1034495, + 1034527, + 1034751, + 1034783, + 1034847, + 1034911, + 1034975, + 1035167, + 1035295, + 1035519, + 1035647, + 1035775, + 1035967, + 1035999, + 1036031, + 1036575, + 1036671, + 1036735, + 1036767, + 1037119, + 1037279, + 1037343, + 1037471, + 1037567, + 1037663, + 1037695, + 1037855, + 1037983, + 1038175, + 1038687, + 1038751, + 1038783, + 1038879, + 1038911, + 1038975, + 1039071, + 1039103, + 1039135, + 1039263, + 1039295, + 1039327, + 1039455, + 1039615, + 1039647, + 1039839, + 1039871, + 1039903, + 1040031, + 1040383, + 1040447, + 1040639, + 1040767, + 1040799, + 1040863, + 1041279, + 1041343, + 1041407, + 1041503, + 1041727, + 1041791, + 1041951, + 1042111, + 1042239, + 1042271, + 1042591, + 1042783, + 1042847, + 1042911, + 1043231, + 1043263, + 1043455, + 1043487, + 1043615, + 1043647, + 1043679, + 1043999, + 1044031, + 1044095, + 1044127, + 1044191, + 1044447, + 1044639, + 1044703, + 1045087, + 1045215, + 1045311, + 1045503, + 1045535, + 1045727, + 1046239, + 1046431, + 1046463, + 1046687, + 1046751, + 1046815, + 1046879, + 1046975, + 1047007, + 1047359, + 1047391, + 1047455, + 1047551, + 1047615, + 1047743, + 1047807, + 1047903, + 1047967, + 1048031, + 1048351, + 1048799, + 1048831, + 1048959, + 1049375, + 1049407, + 1049439, + 1049535, + 1049631, + 1049727, + 1049759, + 1049791, + 1049887, + 1050143, + 1050527, + 1050655, + 1050815, + 1051039, + 1051071, + 1051135, + 1051295, + 1051327, + 1051391, + 1051455, + 1051487, + 1051519, + 1051647, + 1051871, + 1051935, + 1051967, + 1052095, + 1052223, + 1052575, + 1052639, + 1052799, + 1052831, + 1052895, + 1053183, + 1053279, + 1053439, + 1053663, + 1053791, + 1053855, + 1054079, + 1054239, + 1054271, + 1054303, + 1054335, + 1054495, + 1054783, + 1055007, + 1055231, + 1055327, + 1055423, + 1055647, + 1055711, + 1055743, + 1055775, + 1055903, + 1056191, + 1056255, + 1056511, + 1056607, + 1056767, + 1056799, + 1057087, + 1057279, + 1057503, + 1057567, + 1057727, + 1057791, + 1057887, + 1057983, + 1058271, + 1058719, + 1058783, + 1058911, + 1059359, + 1059423, + 1059487, + 1059743, + 1059871, + 1060063, + 1060223, + 1060255, + 1060319, + 1060415, + 1060735, + 1060767, + 1060799, + 1060895, + 1060959, + 1061151, + 1061375, + 1061599, + 1061791, + 1061823, + 1061983, + 1062239, + 1063391, + 1063615, + 1063711, + 1063839, + 1063935, + 1063967, + 1064287, + 1064319, + 1064511, + 1064703, + 1064959, + 1065119, + 1065823, + 1065951, + 1066047, + 1066207, + 1066239, + 1066335, + 1066399, + 1066719, + 1066751, + 1066783, + 1066847, + 1066975, + 1067071, + 1067167, + 1067231, + 1067455, + 1067519, + 1067903, + 1068095, + 1068223, + 1068319, + 1068415, + 1068447, + 1068479, + 1068895, + 1069055, + 1069119, + 1069375, + 1069503, + 1069759, + 1069823, + 1069887, + 1070047, + 1070399, + 1070463, + 1070751, + 1070783, + 1070815, + 1071103, + 1071423, + 1071487, + 1071551, + 1071647, + 1071711, + 1071839, + 1072063, + 1072223, + 1072415, + 1072447, + 1072543, + 1072991, + 1073087, + 1073151, + 1073375, + 1073407, + 1073439, + 1073983, + 1074079, + 1074399, + 1074431, + 1074559, + 1074719, + 1074783, + 1074911, + 1075071, + 1075135, + 1075231, + 1075263, + 1075295, + 1075359, + 1075423, + 1075487, + 1075615, + 1075743, + 1075807, + 1076031, + 1076127, + 1076191, + 1076255, + 1076447, + 1076703, + 1076863, + 1076895, + 1077119, + 1077439, + 1077567, + 1077663, + 1077727, + 1077887, + 1077983, + 1078015, + 1078335, + 1078367, + 1078527, + 1078591, + 1079071, + 1079167, + 1079455, + 1079487, + 1079583, + 1079679, + 1079807, + 1079903, + 1080063, + 1080159, + 1080223, + 1080383, + 1080511, + 1080703, + 1080767, + 1080799, + 1081215, + 1081247, + 1081311, + 1081343, + 1081375, + 1081471, + 1081887, + 1082143, + 1082303, + 1082623, + 1082975, + 1083103, + 1083199, + 1083231, + 1083327, + 1083455, + 1083583, + 1083615, + 1083935, + 1083999, + 1084191, + 1084319, + 1084447, + 1084927, + 1084991, + 1085279, + 1085311, + 1085375, + 1085407, + 1085439, + 1085599, + 1085727, + 1085791, + 1085919, + 1085951, + 1085983, + 1086047, + 1086207, + 1086591, + 1086623, + 1086687, + 1086751, + 1086943, + 1087103, + 1087135, + 1087487, + 1087519, + 1087647, + 1087679, + 1087967, + 1088127, + 1088223, + 1088287, + 1088415, + 1088639, + 1088703, + 1088767, + 1088799, + 1088863, + 1089023, + 1089151, + 1089279, + 1089311, + 1089343, + 1089407, + 1089503, + 1089695, + 1090111, + 1090175, + 1090271, + 1090303, + 1090463, + 1090527, + 1090591, + 1090623, + 1090719, + 1090879, + 1090911, + 1090975, + 1091391, + 1091487, + 1091583, + 1091679, + 1091711, + 1091743, + 1091839, + 1091903, + 1092159, + 1092191, + 1092415, + 1092447, + 1092671, + 1092927, + 1092959, + 1093055, + 1093279, + 1093439, + 1093759, + 1093919, + 1094079, + 1094143, + 1094303, + 1094463, + 1094495, + 1094687, + 1094719, + 1094751, + 1094783, + 1094943, + 1095135, + 1095359, + 1095455, + 1095647, + 1095775, + 1095807, + 1095871, + 1096063, + 1096095, + 1096159, + 1096351, + 1096703, + 1096767, + 1097119, + 1097183, + 1097279, + 1097407, + 1097439, + 1097503, + 1097727, + 1097759, + 1097791, + 1098175, + 1098239, + 1098335, + 1098367, + 1098623, + 1098719, + 1098975, + 1099039, + 1099167, + 1099199, + 1099455, + 1099487, + 1099679, + 1099839, + 1099935, + 1099999, + 1100127, + 1100191, + 1100415, + 1100799, + 1100927, + 1101119, + 1101151, + 1101183, + 1101311, + 1101503, + 1101759, + 1101887, + 1101919, + 1101951, + 1102175, + 1102271, + 1102303, + 1102367, + 1102463, + 1102943, + 1103167, + 1103775, + 1103903, + 1104191, + 1104223, + 1104447, + 1104671, + 1104991, + 1105119, + 1105247, + 1105343, + 1105375, + 1105407, + 1105503, + 1105983, + 1106079, + 1106175, + 1106207, + 1106527, + 1106655, + 1106783, + 1107007, + 1107071, + 1107167, + 1107199, + 1108063, + 1108127, + 1108511, + 1108575, + 1109023, + 1109151, + 1109471, + 1109663, + 1109695, + 1109727, + 1109791, + 1109823, + 1110111, + 1110143, + 1110239, + 1110431, + 1110783, + 1111039, + 1111167, + 1111487, + 1111519, + 1111551, + 1112063, + 1112127, + 1112287, + 1112511, + 1112639, + 1112991, + 1113023, + 1113247, + 1113407, + 1113503, + 1113567, + 1113599, + 1113663, + 1113983, + 1114015, + 1114047, + 1114207, + 1114399, + 1114431, + 1114559, + 1114623, + 1115071, + 1115135, + 1115807, + 1116223, + 1116671, + 1116703, + 1116735, + 1116767, + 1117375, + 1117471, + 1117567, + 1117599, + 1117727, + 1117823, + 1117855, + 1117951, + 1117983, + 1118079, + 1118239, + 1118719, + 1119423, + 1119551, + 1119583, + 1119615, + 1119999, + 1120383, + 1120511, + 1120543, + 1120671, + 1121087, + 1121151, + 1121279, + 1121503, + 1121663, + 1121919, + 1121951, + 1122047, + 1122751, + 1123007, + 1123039, + 1123167, + 1123199, + 1123231, + 1123391, + 1123551, + 1123583, + 1123807, + 1123839, + 1123903, + 1123935, + 1124063, + 1124127, + 1124159, + 1124255, + 1124639, + 1124799, + 1124863, + 1124959, + 1124991, + 1125087, + 1125151, + 1125215, + 1125311, + 1125471, + 1125823, + 1125855, + 1126143, + 1126527, + 1126719, + 1126751, + 1126943, + 1127167, + 1127199, + 1127711, + 1127871, + 1128159, + 1128223, + 1128319, + 1128415, + 1128447, + 1128479, + 1128511, + 1128703, + 1128927, + 1128991, + 1129247, + 1129855, + 1129951, + 1130303, + 1130431, + 1130623, + 1130751, + 1130879, + 1130975, + 1131007, + 1131071, + 1131103, + 1131231, + 1131615, + 1131903, + 1131935, + 1131967, + 1132255, + 1132607, + 1132671, + 1132831, + 1132927, + 1133599, + 1133631, + 1133663, + 1133919, + 1133951, + 1134271, + 1134591, + 1134655, + 1135071, + 1135103, + 1135199, + 1135231, + 1135327, + 1135391, + 1135423, + 1135647, + 1135775, + 1135807, + 1136127, + 1136607, + 1136735, + 1136799, + 1136959, + 1137087, + 1137215, + 1137311, + 1137471, + 1137983, + 1138431, + 1138463, + 1138719, + 1138879, + 1139007, + 1139103, + 1139167, + 1139231, + 1139487, + 1139647, + 1140127, + 1140447, + 1140639, + 1140863, + 1141023, + 1141087, + 1141215, + 1141567, + 1141695, + 1141919, + 1142047, + 1142143, + 1142271, + 1142431, + 1142687, + 1142783, + 1142911, + 1143071, + 1143487, + 1143935, + 1144319, + 1144351, + 1144447, + 1144543, + 1144671, + 1144927, + 1144959, + 1145087, + 1145311, + 1145407, + 1145631, + 1145695, + 1145951, + 1146111, + 1146271, + 1146495, + 1146751, + 1146847, + 1147007, + 1147199, + 1147711, + 1147839, + 1148031, + 1148127, + 1148319, + 1148383, + 1148511, + 1149087, + 1149439, + 1149567, + 1150207, + 1150271, + 1150367, + 1150719, + 1150975, + 1151231, + 1151551, + 1151903, + 1152255, + 1152351, + 1152447, + 1152607, + 1152863, + 1153279, + 1153311, + 1153407, + 1153567, + 1153663, + 1153695, + 1153727, + 1153919, + 1153983, + 1154239, + 1154271, + 1154399, + 1154815, + 1154911, + 1155167, + 1155487, + 1155679, + 1156127, + 1156191, + 1156319, + 1156383, + 1157023, + 1157087, + 1157119, + 1157247, + 1157343, + 1157823, + 1157951, + 1157983, + 1158591, + 1159423, + 1159551, + 1160063, + 1160415, + 1160511, + 1160543, + 1161151, + 1161567, + 1161695, + 1162143, + 1162527, + 1162719, + 1162783, + 1162847, + 1163103, + 1163775, + 1163807, + 1163903, + 1163967, + 1164319, + 1164863, + 1165119, + 1165535, + 1165695, + 1165791, + 1166335, + 1166399, + 1167039, + 1167391, + 1167487, + 1167743, + 1168031, + 1168543, + 1168639, + 1169407, + 1169663, + 1169727, + 1170175, + 1170239, + 1170367, + 1170687, + 1171167, + 1171295, + 1171455, + 1172031, + 1172607, + 1172671, + 1172831, + 1172991, + 1173151, + 1173439, + 1173535, + 1173567, + 1173887, + 1174175, + 1174559, + 1174655, + 1174879, + 1174943, + 1175039, + 1175487, + 1175775, + 1175903, + 1175967, + 1176223, + 1176415, + 1176895, + 1177503, + 1177759, + 1177791, + 1178111, + 1178175, + 1178335, + 1178751, + 1178783, + 1179487, + 1180351, + 1181311, + 1181375, + 1181407, + 1181503, + 1181535, + 1181567, + 1181695, + 1181919, + 1182431, + 1182559, + 1182591, + 1182847, + 1182879, + 1183007, + 1183359, + 1183583, + 1183807, + 1184063, + 1184415, + 1185759, + 1185823, + 1185855, + 1185887, + 1186079, + 1186239, + 1186687, + 1187423, + 1187711, + 1187871, + 1187903, + 1188159, + 1188383, + 1188479, + 1188607, + 1190367, + 1190527, + 1190591, + 1191231, + 1191711, + 1192095, + 1192127, + 1192671, + 1192703, + 1192927, + 1193087, + 1193151, + 1193471, + 1193855, + 1194303, + 1194751, + 1194783, + 1195199, + 1195263, + 1195391, + 1195455, + 1196255, + 1196703, + 1197183, + 1197407, + 1197471, + 1198047, + 1198335, + 1198911, + 1199199, + 1199775, + 1199999, + 1200575, + 1200607, + 1201119, + 1201279, + 1201791, + 1202143, + 1202335, + 1202463, + 1202623, + 1202783, + 1202815, + 1202943, + 1202975, + 1203231, + 1203391, + 1203487, + 1203615, + 1203935, + 1204255, + 1204447, + 1204543, + 1204831, + 1204959, + 1204991, + 1205599, + 1205695, + 1206271, + 1206367, + 1207263, + 1207615, + 1207711, + 1207775, + 1207807, + 1208223, + 1208383, + 1209119, + 1209151, + 1209183, + 1209407, + 1209791, + 1209983, + 1210207, + 1210399, + 1210527, + 1210687, + 1211071, + 1211391, + 1211615, + 1211839, + 1211935, + 1212159, + 1212223, + 1212639, + 1212831, + 1213087, + 1213151, + 1213695, + 1213983, + 1214367, + 1215103, + 1215455, + 1215775, + 1215839, + 1215999, + 1216479, + 1217087, + 1217247, + 1217311, + 1217407, + 1217503, + 1217951, + 1218143, + 1218943, + 1219071, + 1219231, + 1219391, + 1219935, + 1220031, + 1220063, + 1222207, + 1222623, + 1222719, + 1223071, + 1223263, + 1223391, + 1223743, + 1223775, + 1223839, + 1224767, + 1225119, + 1225535, + 1226399, + 1227647, + 1228671, + 1228703, + 1228831, + 1228927, + 1229087, + 1229471, + 1229535, + 1229983, + 1230463, + 1230623, + 1230847, + 1231583, + 1231839, + 1232447, + 1232607, + 1232767, + 1233087, + 1233599, + 1234111, + 1234143, + 1234335, + 1234367, + 1234463, + 1234911, + 1235455, + 1235647, + 1236159, + 1237599, + 1237727, + 1237919, + 1238239, + 1238271, + 1239967, + 1240191, + 1240319, + 1240607, + 1240735, + 1240991, + 1241503, + 1241599, + 1241983, + 1242047, + 1242079, + 1242111, + 1242495, + 1242719, + 1243359, + 1243775, + 1243839, + 1244095, + 1244415, + 1244639, + 1244991, + 1245279, + 1245503, + 1245855, + 1245887, + 1246111, + 1246175, + 1246527, + 1246559, + 1246783, + 1247583, + 1247967, + 1248447, + 1248607, + 1248799, + 1249599, + 1250271, + 1250687, + 1250719, + 1251103, + 1251647, + 1251775, + 1251967, + 1252063, + 1252895, + 1253023, + 1253119, + 1254271, + 1254687, + 1255071, + 1255647, + 1255743, + 1255775, + 1255967, + 1256255, + 1256319, + 1256671, + 1256927, + 1257023, + 1257119, + 1257183, + 1257535, + 1257663, + 1257727, + 1257855, + 1258431, + 1258783, + 1259231, + 1259839, + 1260191, + 1260255, + 1260575, + 1260607, + 1260703, + 1262335, + 1262431, + 1262687, + 1263231, + 1263455, + 1263743, + 1264255, + 1264383, + 1264671, + 1264831, + 1264959, + 1265087, + 1265791, + 1266079, + 1266239, + 1266591, + 1266623, + 1266975, + 1267359, + 1267807, + 1268255, + 1268607, + 1270271, + 1270911, + 1271135, + 1271583, + 1271871, + 1272319, + 1272447, + 1272959, + 1272991, + 1273151, + 1273407, + 1274079, + 1274239, + 1274431, + 1274943, + 1275135, + 1275519, + 1275615, + 1275999, + 1276383, + 1276703, + 1276959, + 1276991, + 1277183, + 1277407, + 1277887, + 1278015, + 1278431, + 1278751, + 1280095, + 1280127, + 1280191, + 1280543, + 1281983, + 1282175, + 1282591, + 1283135, + 1283487, + 1283935, + 1285279, + 1286591, + 1286911, + 1287103, + 1288095, + 1288447, + 1288607, + 1288671, + 1289471, + 1289983, + 1290047, + 1290367, + 1291263, + 1291519, + 1292255, + 1292767, + 1293247, + 1293791, + 1293951, + 1294879, + 1294975, + 1295263, + 1295551, + 1297055, + 1299295, + 1299519, + 1299551, + 1299615, + 1299743, + 1300735, + 1300799, + 1301215, + 1302399, + 1302783, + 1302975, + 1303167, + 1303967, + 1304671, + 1305215, + 1305471, + 1305791, + 1306463, + 1306527, + 1306559, + 1306719, + 1306751, + 1307071, + 1307839, + 1308287, + 1309087, + 1309407, + 1309855, + 1310399, + 1310943, + 1312031, + 1312351, + 1312639, + 1312799, + 1313215, + 1313791, + 1315071, + 1315135, + 1315327, + 1315679, + 1315967, + 1316031, + 1316127, + 1316351, + 1316639, + 1316831, + 1317727, + 1318111, + 1318207, + 1318431, + 1318527, + 1318591, + 1318751, + 1318783, + 1319135, + 1319455, + 1319711, + 1321503, + 1321535, + 1321951, + 1322015, + 1322815, + 1323007, + 1323519, + 1323679, + 1325247, + 1325279, + 1325407, + 1325695, + 1325727, + 1327167, + 1327871, + 1327903, + 1328319, + 1328863, + 1329247, + 1329631, + 1330111, + 1331199, + 1331231, + 1332927, + 1333279, + 1333695, + 1333951, + 1334463, + 1334815, + 1335359, + 1335487, + 1335807, + 1336479, + 1336511, + 1337279, + 1337599, + 1337983, + 1338655, + 1338719, + 1338815, + 1340319, + 1340639, + 1341759, + 1342943, + 1343071, + 1343327, + 1343423, + 1343487, + 1343583, + 1344095, + 1344383, + 1344415, + 1344543, + 1344767, + 1344831, + 1345439, + 1345535, + 1346079, + 1346431, + 1346495, + 1346591, + 1346719, + 1347199, + 1347327, + 1347455, + 1347903, + 1350015, + 1350879, + 1350943, + 1351231, + 1352255, + 1352863, + 1353023, + 1353119, + 1354751, + 1354815, + 1354847, + 1355423, + 1355871, + 1356415, + 1357567, + 1358719, + 1358815, + 1358879, + 1359903, + 1360607, + 1360703, + 1361695, + 1362111, + 1363167, + 1363775, + 1363967, + 1364927, + 1365759, + 1365823, + 1366079, + 1366623, + 1367039, + 1367679, + 1367743, + 1369087, + 1369535, + 1370015, + 1370271, + 1371391, + 1371455, + 1372351, + 1372639, + 1372831, + 1373375, + 1373663, + 1373919, + 1374143, + 1374591, + 1374623, + 1374943, + 1374975, + 1375551, + 1376063, + 1376607, + 1376927, + 1377279, + 1377471, + 1377919, + 1378367, + 1378751, + 1379103, + 1380831, + 1381311, + 1382143, + 1383423, + 1383455, + 1383519, + 1384127, + 1384159, + 1384383, + 1385055, + 1385503, + 1386175, + 1386239, + 1386591, + 1387647, + 1388607, + 1388831, + 1388991, + 1389215, + 1389439, + 1389791, + 1389983, + 1390847, + 1391711, + 1393407, + 1393535, + 1393791, + 1394175, + 1394303, + 1394879, + 1396479, + 1396511, + 1397087, + 1399103, + 1399839, + 1399903, + 1400831, + 1401055, + 1402143, + 1402623, + 1403327, + 1403519, + 1404575, + 1406015, + 1406975, + 1407103, + 1407263, + 1407647, + 1407903, + 1408895, + 1409215, + 1409503, + 1409567, + 1410367, + 1411391, + 1411455, + 1411871, + 1412127, + 1412511, + 1413759, + 1413983, + 1414143, + 1414463, + 1414847, + 1415967, + 1417023, + 1417375, + 1417983, + 1418015, + 1418463, + 1418943, + 1419103, + 1420799, + 1421247, + 1421311, + 1423295, + 1423551, + 1424095, + 1424159, + 1426623, + 1427807, + 1427871, + 1428255, + 1428927, + 1429407, + 1429823, + 1429951, + 1430271, + 1432447, + 1432895, + 1432927, + 1433567, + 1434079, + 1434335, + 1435039, + 1435327, + 1438847, + 1439455, + 1439647, + 1440255, + 1440991, + 1442303, + 1442943, + 1443135, + 1444031, + 1444639, + 1444671, + 1445215, + 1446047, + 1447775, + 1447807, + 1448351, + 1449375, + 1449727, + 1451135, + 1452383, + 1452575, + 1453247, + 1453663, + 1454719, + 1458143, + 1458655, + 1459199, + 1460351, + 1462431, + 1464095, + 1465215, + 1466623, + 1468511, + 1468767, + 1469695, + 1470911, + 1473183, + 1473471, + 1474303, + 1474559, + 1475135, + 1475199, + 1476831, + 1477375, + 1478655, + 1478847, + 1480479, + 1483487, + 1484351, + 1485439, + 1485663, + 1485695, + 1487199, + 1487615, + 1487839, + 1488479, + 1488959, + 1489375, + 1491839, + 1492447, + 1495711, + 1496191, + 1497695, + 1498559, + 1498815, + 1501599, + 1501919, + 1501951, + 1503295, + 1504127, + 1505247, + 1507231, + 1511999, + 1512287, + 1514495, + 1515007, + 1519519, + 1522015, + 1522879, + 1523007, + 1524479, + 1526623, + 1526847, + 1528191, + 1528287, + 1528607, + 1531167, + 1531551, + 1533631, + 1534079, + 1535711, + 1536543, + 1537919, + 1539135, + 1539295, + 1539839, + 1540991, + 1541599, + 1542751, + 1544319, + 1547679, + 1547871, + 1548031, + 1548543, + 1549983, + 1551327, + 1551679, + 1551711, + 1552479, + 1552511, + 1553119, + 1553151, + 1555903, + 1556191, + 1556895, + 1557503, + 1558079, + 1559199, + 1559263, + 1560767, + 1561055, + 1562815, + 1563935, + 1566079, + 1570175, + 1570719, + 1570751, + 1572063, + 1572095, + 1573919, + 1574239, + 1574975, + 1575583, + 1576031, + 1578047, + 1578431, + 1578655, + 1580703, + 1581759, + 1583711, + 1584255, + 1585887, + 1586815, + 1589087, + 1594431, + 1595647, + 1596319, + 1596799, + 1596831, + 1597343, + 1598175, + 1599039, + 1599071, + 1599327, + 1600511, + 1602175, + 1603231, + 1603807, + 1604511, + 1605023, + 1606047, + 1607135, + 1607391, + 1609375, + 1609439, + 1610527, + 1613535, + 1614239, + 1616735, + 1617567, + 1617823, + 1619711, + 1620287, + 1620415, + 1621535, + 1622623, + 1623615, + 1628575, + 1629023, + 1629727, + 1632511, + 1635295, + 1637663, + 1639071, + 1639103, + 1639231, + 1641055, + 1641599, + 1642367, + 1642431, + 1643935, + 1644511, + 1645343, + 1647999, + 1649247, + 1649983, + 1651103, + 1651615, + 1655743, + 1657567, + 1658431, + 1662687, + 1664319, + 1666975, + 1668063, + 1669375, + 1670143, + 1672159, + 1673471, + 1673855, + 1676191, + 1676543, + 1676703, + 1678719, + 1681119, + 1681215, + 1682623, + 1686815, + 1687551, + 1689695, + 1690047, + 1690687, + 1691871, + 1693055, + 1694431, + 1696639, + 1696895, + 1698655, + 1699455, + 1699615, + 1701695, + 1702591, + 1704543, + 1709215, + 1710879, + 1711231, + 1714047, + 1715519, + 1716127, + 1718559, + 1718623, + 1718783, + 1719583, + 1719807, + 1720415, + 1722719, + 1723487, + 1724223, + 1725375, + 1726335, + 1727007, + 1729055, + 1730943, + 1736383, + 1739519, + 1739615, + 1741055, + 1743007, + 1743967, + 1751103, + 1751615, + 1752319, + 1752927, + 1753119, + 1754559, + 1756063, + 1757407, + 1758399, + 1760831, + 1761439, + 1762431, + 1762559, + 1762879, + 1764511, + 1768383, + 1771583, + 1772671, + 1772959, + 1774303, + 1776991, + 1777215, + 1778367, + 1779007, + 1782559, + 1783103, + 1783263, + 1784511, + 1786079, + 1792031, + 1796543, + 1798015, + 1800671, + 1802431, + 1806783, + 1808511, + 1811071, + 1812927, + 1813215, + 1814655, + 1815711, + 1818015, + 1818239, + 1821695, + 1823711, + 1824511, + 1825695, + 1826175, + 1829183, + 1829823, + 1831391, + 1832639, + 1833055, + 1834879, + 1834975, + 1835103, + 1835551, + 1837695, + 1837983, + 1840479, + 1840959, + 1842815, + 1844287, + 1844447, + 1847615, + 1850079, + 1852319, + 1852351, + 1852575, + 1852895, + 1856607, + 1858047, + 1858783, + 1858847, + 1860447, + 1864607, + 1865055, + 1868607, + 1870943, + 1871615, + 1875167, + 1875999, + 1876543, + 1878207, + 1878591, + 1880767, + 1882335, + 1883167, + 1884575, + 1884703, + 1885951, + 1887967, + 1887999, + 1890047, + 1890751, + 1891135, + 1896639, + 1896991, + 1898207, + 1899999, + 1902911, + 1904255, + 1906367, + 1907199, + 1915135, + 1920031, + 1920671, + 1921407, + 1923807, + 1926815, + 1929247, + 1931423, + 1933887, + 1938751, + 1939839, + 1942143, + 1942527, + 1942975, + 1946591, + 1949087, + 1949215, + 1949375, + 1949951, + 1950047, + 1956703, + 1960319, + 1961919, + 1968447, + 1970623, + 1972255, + 1974399, + 1976639, + 1979039, + 1979615, + 1980575, + 1981247, + 1982719, + 1984031, + 1984959, + 1991391, + 1991615, + 1991967, + 1994303, + 1996735, + 1998719, + 1999359, + 2002975, + 2004767, + 2005407, + 2005759, + 2010047, + 2013119, + 2013695, + 2013983, + 2014879, + 2017663, + 2018751, + 2022815, + 2025823, + 2026335, + 2026591, + 2027967, + 2034463, + 2034527, + 2035327, + 2037983, + 2045535, + 2051007, + 2052575, + 2054207, + 2055231, + 2059967, + 2061727, + 2061887, + 2064703, + 2067679, + 2074335, + 2074463, + 2074911, + 2079583, + 2079615, + 2080127, + 2082015, + 2086271, + 2098463, + 2099103, + 2101567, + 2102559, + 2103199, + 2107999, + 2108063, + 2108671, + 2110239, + 2111583, + 2113535, + 2114047, + 2116191, + 2123615, + 2126431, + 2132415, + 2135327, + 2136671, + 2140063, + 2140895, + 2141791, + 2150847, + 2155135, + 2158751, + 2161279, + 2163711, + 2165439, + 2168319, + 2169471, + 2170431, + 2170719, + 2174207, + 2175071, + 2177151, + 2177823, + 2184767, + 2187231, + 2188063, + 2190143, + 2191135, + 2192639, + 2194879, + 2196767, + 2201279, + 2204415, + 2206239, + 2210175, + 2214303, + 2217759, + 2218367, + 2219999, + 2230239, + 2231615, + 2236639, + 2239455, + 2241087, + 2242783, + 2243583, + 2246207, + 2247199, + 2248351, + 2254207, + 2261599, + 2264127, + 2264767, + 2264959, + 2267199, + 2268799, + 2269311, + 2271423, + 2280031, + 2280511, + 2286783, + 2287359, + 2289823, + 2293951, + 2294079, + 2296383, + 2302527, + 2307839, + 2310911, + 2316415, + 2325055, + 2326911, + 2334943, + 2340639, + 2342751, + 2344255, + 2347423, + 2351167, + 2352895, + 2360127, + 2360703, + 2371935, + 2374175, + 2375455, + 2376191, + 2378047, + 2381567, + 2386047, + 2391871, + 2394815, + 2398047, + 2402975, + 2411007, + 2412095, + 2422751, + 2424191, + 2424927, + 2429439, + 2435199, + 2436319, + 2441535, + 2447103, + 2456447, + 2460159, + 2461791, + 2469311, + 2478879, + 2485471, + 2488639, + 2494463, + 2495135, + 2503711, + 2504991, + 2506815, + 2507135, + 2508191, + 2514623, + 2516127, + 2520799, + 2528735, + 2531583, + 2535487, + 2541503, + 2542047, + 2544575, + 2549407, + 2551455, + 2558271, + 2560479, + 2572511, + 2579775, + 2582815, + 2584191, + 2594079, + 2602431, + 2605279, + 2605343, + 2614783, + 2628639, + 2629951, + 2668959, + 2672383, + 2674911, + 2680927, + 2684095, + 2697439, + 2721023, + 2736927, + 2737695, + 2737919, + 2748735, + 2755615, + 2756927, + 2757343, + 2769471, + 2775551, + 2785023, + 2785727, + 2790559, + 2792831, + 2802303, + 2817343, + 2820191, + 2820351, + 2838591, + 2844511, + 2848383, + 2854911, + 2864543, + 2865183, + 2885471, + 2890143, + 2895999, + 2899231, + 2916319, + 2919295, + 2921279, + 2938399, + 2957823, + 2966591, + 3004671, + 3012703, + 3014847, + 3033407, + 3063039, + 3064863, + 3068671, + 3073759, + 3084287, + 3110015, + 3130527, + 3190431, + 3196703, + 3228799, + 3255647, + 3287039, + 3312959, + 3321471, + 3331807, + 3339743, + 3345919, + 3354271, + 3357311, + 3381119, + 3408959, + 3411423, + 3451839, + 3457727, + 3504831, + 3519743, + 3536575, + 3557503, + 3579039, + 3601567, + 3602719, + 3607039, + 3616223, + 3625567, + 3637375, + 3639103, + 3660159, + 3697503, + 3729823, + 3731487, + 3740991, + 3781087, + 3796511, + 3806463, + 3807807, + 3852799, + 3862975, + 3908223, + 3980831, + 4142335, + 4157471, + 4168991, + 4176255 + ] +} diff --git a/baremetal/parseTestResultIntoCsv.py b/baremetal/parseTestResultIntoCsv.py new file mode 100755 index 0000000000..92ac44ddc7 --- /dev/null +++ b/baremetal/parseTestResultIntoCsv.py @@ -0,0 +1,148 @@ +#!/usr/bin/python3 + +import sys +import json +import re +import os +import pandas as pd +from csv import writer + +''' +Parsing support for the following command: +sudo time -v numactl -N 0 \ + perf stat -e dsa0/event=0x1,event_category=0x0/,\ + dsa2/event=0x1,event_category=0x0/,\ + dsa4/event=0x1,event_category=0x0/,\ + dsa6/event=0x1,event_category=0x0/,\ + dsa8/event=0x1,event_category=0x0/ \ + opt/cachelib/bin/cachebench --json_test_config \ + --report_api_latency +''' +def num(s): + try: + return int(s) + except ValueError: + return float(s) + + +def parse(fileName, testName): + with open(fileName) as file: + data = file.readlines() + + ## Parse config + config = [] + seen = False + for line in data: + line.strip() + if line == "{\n": + seen = True + elif line == "Welcome to OSS version of cachebench\n": + break + elif "reading distribution params from" in line or line == '\n': + continue + if seen: + config.append(line) + + jconfig = json.loads("".join(config)) + + ## Split cachebench and system results + cbResults = [] + sysResults = [] + seen = 0 + for line in data: + line.strip() + if "== Test Results ==" in line: + seen = 1 + continue + elif "Performance counter stats for 'system wide'" in line: + seen = 2 + continue + elif "== KVReplayGenerator Stats ==" in line: + seen = 0 + continue + + if seen == 1: + cbResults.append(line) + elif seen == 2: + sysResults.append(line) + + ## Parse cachebench metrics + cbMetrics = {} + for line in cbResults: + if ':' in line: + key, value, *_ = [x.rstrip(', \n').replace(" ","_").lower() for x in line.split(':') if not len(x) == 2] + key = re.sub('__+' , '_', key) + key = re.sub('[^\d\w]', '', key) + + value = re.sub('[^0-9.]', '', value) + value = num(value) + + cbMetrics[key] = value + + #cbMetrics = {'cachebench_metrics': cbMetrics} + + ## Parse system metrics + sysMetrics = {} + for line in sysResults: + if "Elapsed (wall clock) time" in line: + continue + + elif "numactl" in line: + value = line.split(':', 1)[-1].strip().replace('"','') + sysMetrics["command"] = value + + elif "seconds time elapsed" in line: + value, key = [x.strip().lower() for x in line.split('seconds')] + sysMetrics["time_elapsed_in_secs"] = num(value.strip()) + + elif ",event_category=0x" in line: + value, key = [x.strip().lower() for x in line.split('dsa')] + key = "dsa" + key.strip() + sysMetrics[key] = num(value.strip()) + + elif ':' in line: + key, value, *_ = [x.rstrip(', \n').replace(" ","_").lower() for x in line.split(':') if not len(x) == 2] + key = re.sub('__+' , '_', key) + key = re.sub('[^\d\w]', '', key) + value = re.sub('[^0-9.]', '', value) + value = num(value) + sysMetrics[key] = value + + #sysMetrics = {'system_metrics': sysMetrics} + + ## Create the unified view + #joined = {**jconfig, **cbMetrics, **sysMetrics} + joined = {**cbMetrics} + label = {"label" : testName} + joined.update(label) + return joined + #return json.dumps(joined, indent=2) + + +def main(): + args = sys.argv[1:] + if len(args) != 3: + print("Invalid Args. Required : file-name, test-name, out-csv") + exit() + + fileName = args[0] + testName = args[1] + outFile = args[2] + + keys = [] + values = [] + out = parse(fileName, testName) + for key in out: + keys.append(key) + values.append(str(out[key])) + + addIndex = not os.path.isfile(outFile) + with open(outFile, 'a') as f: + writer_object = writer(f) + if addIndex: + writer_object.writerow(keys) + writer_object.writerow(values) + f.close() + +if __name__ == '__main__': + main() diff --git a/baremetal/runTestAndCovertToJson.py b/baremetal/runTestAndCovertToJson.py new file mode 100755 index 0000000000..49dc87b642 --- /dev/null +++ b/baremetal/runTestAndCovertToJson.py @@ -0,0 +1,201 @@ +#!/usr/bin/python3 + +import os +import sys +import subprocess +import re +import json +import pandas as pd +import time + +gDsaDevList = ["0", "2", "4", "6"] + +# Convert to int +def num(s): + try: + return int(s) + except ValueError: + return float(s) + + +# Parse the result +def parseResult(outputPath): + with open(outputPath) as file: + data = file.readlines() + + ## Parse config + config = [] + seen = False + for line in data: + line.strip() + if line == "{\n": + seen = True + elif line == "Welcome to OSS version of cachebench\n": + break + elif "reading distribution params from" in line or line == '\n': + continue + if seen: + config.append(line) + + jconfig = json.loads("".join(config)) + + ## Split cachebench and system results + cbResults = [] + sysResults = [] + seen = 0 + for line in data: + line.strip() + if "== Test Results ==" in line: + seen = 1 + continue + elif "Performance counter stats for 'system wide'" in line: + seen = 2 + continue + elif "== KVReplayGenerator Stats ==" in line: + seen = 0 + continue + + if seen == 1: + cbResults.append(line) + elif seen == 2: + sysResults.append(line) + + ## Parse cachebench metrics + cbMetrics = {} + for line in cbResults: + if ':' in line: + key, value, *_ = [x.rstrip(', \n').replace(" ","_").lower() for x in line.split(':') if not len(x) == 2] + key = re.sub('__+' , '_', key) + key = re.sub('[^\d\w]', '', key) + + value = re.sub('[^0-9.]', '', value) + value = num(value) + + cbMetrics[key] = value + + cbMetrics = {'cachebench_metrics': cbMetrics} + + ## Parse system metrics + sysMetrics = {} + for line in sysResults: + if "Elapsed (wall clock) time" in line: + continue + + elif "numactl" in line: + value = line.split(':', 1)[-1].strip().replace('"','') + sysMetrics["command"] = value + + elif "seconds time elapsed" in line: + value, key = [x.strip().lower() for x in line.split('seconds')] + sysMetrics["time_elapsed_in_secs"] = num(value.strip()) + + elif ",event_category=0x" in line: + value, key = [x.strip().lower() for x in line.split('dsa')] + key = "dsa" + key.strip() + sysMetrics[key] = num(value.strip()) + + elif ':' in line: + key, value, *_ = [x.rstrip(', \n').replace(" ","_").lower() for x in line.split(':') if not len(x) == 2] + key = re.sub('__+' , '_', key) + key = re.sub('[^\d\w]', '', key) + value = re.sub('[^0-9.]', '', value) + value = num(value) + sysMetrics[key] = value + + sysMetrics = {'system_metrics': sysMetrics} + + ## Save the unified view + dictionary = {**jconfig, **cbMetrics, **sysMetrics} + return json.dumps(dictionary, indent=2) + + +# Update the dsa device setup and cachebench config +def updateConfig(config, dsaDevs, evictors, evictBatch, promoters, promoBatch): + pwd = os.getcwd() + dsaEnabled = str(True if dsaDevs > 0 else False).lower() + for i in range(len(gDsaDevList)): + if i < dsaDevs: + cmd = [os.path.join(pwd, "baremetal/accelConfig.sh"), gDsaDevList[i], "yes"] + subprocess.run(cmd) + else: + cmd = [os.path.join(pwd, "baremetal/accelConfig.sh"), gDsaDevList[i], "no"] + subprocess.run(cmd) + + absPath = os.path.join(pwd, config) + data = [] + with open(absPath, "r") as f: + data = json.load(f) + data["cache_config"]["dsaEnabled"] = dsaEnabled + data["cache_config"]["evictorThreads"] = evictors + data["cache_config"]["maxEvictionBatch"] = evictBatch + data["cache_config"]["promoterThreads"] = promoters + data["cache_config"]["maxPromotionBatch"] = promoBatch + + os.remove(absPath) + with open(absPath, "w") as f: + json.dump(data, f, indent=2) + + +# Run a test +def runTest(config, outputPath): + timeCmd = ["time", "-v"] + numaCmd = ["numactl","-N", "0"] + perfCmd = ["perf", "stat", "-e", + "dsa0/event=0x1,event_category=0x0/,\ + dsa0/event=0x10,event_category=0x1/,\ + dsa0/event=0x2,event_category=0x3/,\ + dsa2/event=0x1,event_category=0x0/,\ + dsa2/event=0x10,event_category=0x1/,\ + dsa2/event=0x2,event_category=0x3/,\ + dsa4/event=0x1,event_category=0x0/,\ + dsa4/event=0x10,event_category=0x1/,\ + dsa4/event=0x2,event_category=0x3/,\ + dsa6/event=0x1,event_category=0x0/,\ + dsa6/event=0x10,event_category=0x1/,\ + dsa6/event=0x2,event_category=0x3/,\ + dsa8/event=0x1,event_category=0x0/,\ + dsa8/event=0x10,event_category=0x1/,\ + dsa8/event=0x2,event_category=0x3/".replace(' \\', '').replace(' ', '')] + cbCmd = ["opt/cachelib/bin/cachebench", "--json_test_config"] + extraFlags = ["--report_api_latency", ">&", outputPath] + recipe = timeCmd + numaCmd + perfCmd + cbCmd + [config] + extraFlags + + with open(outputPath, 'w') as f: + ret = subprocess.run(recipe, stdout=f, stderr=f) + + return ret.check_returncode() + + +# Orchestrate config change, run the test and parse the result +def main(): + args = sys.argv[1:] + if len(args) != 7: + print('Invalid args. Required : config-file, dsa-device-count, evictor-threads, ' + 'evictor-batch-size, promoter-threads, promoter-batch-size, output-path') + exit() + + config = args[0] + dsaDevs = num(args[1]) + evictors = num(args[2]) + evictBatch = num(args[3]) + promoters = num(args[4]) + promoBatch = num(args[5]) + outputPath = args[6] + + updateConfig(config, dsaDevs, evictors, evictBatch, promoters, promoBatch) + + outputTxt = outputPath + ".txt" + runTest(config, outputTxt) + + outputJson = outputPath + ".json" + jsonObject = parseResult(outputTxt) + with open(outputJson, "w") as outfile: + outfile.write(jsonObject) + print("Test {0} complete".format(outputPath)) + + time.sleep(15) + +if __name__ == '__main__': + main() + + diff --git a/baremetal/sample.sh b/baremetal/sample.sh new file mode 100755 index 0000000000..77cd6f00cc --- /dev/null +++ b/baremetal/sample.sh @@ -0,0 +1,127 @@ +#!/bin/bash + + +# Check whether you're in the right directory +CHECK_DIR="baremetal" +if [ ! -d "${CHECK_DIR}" ]; then + echo "You're not in the right directory. You need to be on the base CacheLib directory for the script to work." + exit 1 +fi + +# Check whether CacheLib was installed using contrib/build.sh +if [ ! -f "opt/cachelib/bin/cachebench" ]; then + echo "CacheLib was not installed using contrib/build.sh." + echo "runTestAndCovertToJson.py would need modification otherwise" + exit 1 +fi + +CDN_DIR="${CHECK_DIR}/cdn" +NOW=$( date '+%F_%H:%M:%S' ) +RES_DIR="${CDN_DIR}/data-${NOW}/" +mkdir ${RES_DIR} +echo "Create results directory ${RES_DIR}." + + +# Run tests for DSA = 0 +# Evictor threads = 4, 8, 12, 16 or 20 +# Eviction batch size = 100 +# Promoter threads = 0 +# Promotion batch size = 0 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 4 100 0 0 ${RES_DIR}/dsa0_et04_eb100_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 8 100 0 0 ${RES_DIR}/dsa0_et08_eb100_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 0 0 ${RES_DIR}/dsa0_et12_eb100_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 16 100 0 0 ${RES_DIR}/dsa0_et16_eb100_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 20 100 0 0 ${RES_DIR}/dsa0_et20_eb100_pt00_pb000 + +# Run tests for DSA = 4 +# Evictor threads = 4, 8, 12, 16 or 20 +# Eviction batch size = 100 +# Promoter threads = 0 +# Promotion batch size = 0 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 4 100 0 0 ${RES_DIR}/dsa4_et04_eb100_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 8 100 0 0 ${RES_DIR}/dsa4_et08_eb100_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 0 0 ${RES_DIR}/dsa4_et12_eb100_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 16 100 0 0 ${RES_DIR}/dsa4_et16_eb100_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 20 100 0 0 ${RES_DIR}/dsa4_et20_eb100_pt00_pb000 + +# Aggregate results for eb100_pt00_pb000 +./${CHECK_DIR}/aggregateAndFilterTestResults.py ${RES_DIR} eb100_pt00_pb000 + + +# Run tests for DSA = 0 +# Evictor threads = 12 +# Eviction batch size = 100 +# Promoter threads = 4, 8, 12, 16 or 20 +# Promotion batch size = 100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 4 100 ${RES_DIR}/dsa0_et12_eb100_pt04_pb100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 8 100 ${RES_DIR}/dsa0_et12_eb100_pt08_pb100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 12 100 ${RES_DIR}/dsa0_et12_eb100_pt12_pb100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 16 100 ${RES_DIR}/dsa0_et12_eb100_pt16_pb100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 20 100 ${RES_DIR}/dsa0_et12_eb100_pt20_pb100 + +# Run tests for DSA = 4 +# Evictor threads = 12 +# Eviction batch size = 100 +# Promoter threads = 4, 8, 12, 16 or 20 +# Promotion batch size = 100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 4 100 ${RES_DIR}/dsa4_et12_eb100_pt04_pb100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 8 100 ${RES_DIR}/dsa4_et12_eb100_pt08_pb100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 12 100 ${RES_DIR}/dsa4_et12_eb100_pt12_pb100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 16 100 ${RES_DIR}/dsa4_et12_eb100_pt16_pb100 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 20 100 ${RES_DIR}/dsa4_et12_eb100_pt20_pb100 + +# Aggregate results for pb100 +./${CHECK_DIR}/aggregateAndFilterTestResults.py ${RES_DIR} pb100 + + +# Run tests for DSA = 0 +# Evictor threads = 4, 8, 12, 16 or 20 +# Eviction batch size = 200 +# Promoter threads = 0 +# Promotion batch size = 0 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 4 200 0 0 ${RES_DIR}/dsa0_et04_eb200_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 8 200 0 0 ${RES_DIR}/dsa0_et08_eb200_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 200 0 0 ${RES_DIR}/dsa0_et12_eb200_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 16 200 0 0 ${RES_DIR}/dsa0_et16_eb200_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 20 200 0 0 ${RES_DIR}/dsa0_et20_eb200_pt00_pb000 + +# Run tests for DSA = 4 +# Evictor threads = 4, 8, 12, 16 or 20 +# Eviction batch size = 200 +# Promoter threads = 0 +# Promotion batch size = 0 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 4 200 0 0 ${RES_DIR}/dsa4_et04_eb200_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 8 200 0 0 ${RES_DIR}/dsa4_et08_eb200_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 200 0 0 ${RES_DIR}/dsa4_et12_eb200_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 16 200 0 0 ${RES_DIR}/dsa4_et16_eb200_pt00_pb000 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 20 200 0 0 ${RES_DIR}/dsa4_et20_eb200_pt00_pb000 + +# Aggregate results for eb200_pt00_pb000 +./${CHECK_DIR}/aggregateAndFilterTestResults.py ${RES_DIR} eb200_pt00_pb000 + + +# Run tests for DSA = 0 +# Evictor threads = 12 +# Eviction batch size = 100 +# Promoter threads = 4, 8, 12, 16 or 20 +# Promotion batch size = 200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 4 200 ${RES_DIR}/dsa0_et12_eb100_pt04_pb200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 8 200 ${RES_DIR}/dsa0_et12_eb100_pt08_pb200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 12 200 ${RES_DIR}/dsa0_et12_eb100_pt12_pb200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 16 200 ${RES_DIR}/dsa0_et12_eb100_pt16_pb200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 0 12 100 20 200 ${RES_DIR}/dsa0_et12_eb100_pt20_pb200 + +# Run tests for DSA = 4 +# Evictor threads = 12 +# Eviction batch size = 100 +# Promoter threads = 4, 8, 12, 16 or 20 +# Promotion batch size = 200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 4 200 ${RES_DIR}/dsa4_et12_eb100_pt04_pb200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 8 200 ${RES_DIR}/dsa4_et12_eb100_pt08_pb200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 12 200 ${RES_DIR}/dsa4_et12_eb100_pt12_pb200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 16 200 ${RES_DIR}/dsa4_et12_eb100_pt16_pb200 +./${CHECK_DIR}/runTestAndCovertToJson.py ${CDN_DIR}/config.json 4 12 100 20 200 ${RES_DIR}/dsa4_et12_eb100_pt20_pb200 + +# Aggregate results for pb200 +./${CHECK_DIR}/aggregateAndFilterTestResults.py ${RES_DIR} pb200 +