From 4ed16193ab7d19a86dbc1c64b4895fd5133b8fa1 Mon Sep 17 00:00:00 2001 From: Honza Horak Date: Fri, 27 May 2016 00:19:45 +0200 Subject: [PATCH] A reworked version of #194 These changes are based on the code that from https://github.com/sclorg/mariadb-container. Even if the image is meant to work in OpenShift, it doesn't mean that it shouldn't work outside of OpenShift -- the opposite is true, the image should be usable on Atomic or normal Fedora Server/Workstation/Cloud without any issues as well. --- mariadb/Dockerfile | 56 ++- mariadb/README.md | 219 ++++++---- mariadb/root/etc/my.cnf | 10 + mariadb/root/usr/bin/cgroup-limits | 92 ++++ mariadb/root/usr/bin/container-entrypoint | 2 + mariadb/root/usr/bin/mysqld-master | 1 + mariadb/root/usr/bin/mysqld-slave | 1 + mariadb/root/usr/bin/run-mysqld | 35 ++ mariadb/root/usr/bin/run-mysqld-master | 50 +++ mariadb/root/usr/bin/run-mysqld-slave | 60 +++ mariadb/root/usr/libexec/container-setup | 58 +++ .../share/container-scripts/mysql/common.sh | 162 +++++++ .../share/container-scripts/mysql/helpers.sh | 24 + .../mysql/my-base.cnf.template | 5 + .../mysql/my-master.cnf.template | 7 + .../mysql/my-paas.cnf.template | 26 ++ .../mysql/my-repl-gtid.cnf.template | 4 + .../mysql/my-slave.cnf.template | 7 + .../mysql/my-tuning.cnf.template | 28 ++ .../container-scripts/mysql/passwd-change.sh | 25 ++ .../container-scripts/mysql/post-init.sh | 0 .../share/container-scripts/mysql/scl_enable | 3 + .../mysql/validate-replication-variables.sh | 18 + .../mysql/validate-variables.sh | 76 ++++ mariadb/scripts/config_mariadb.sh | 49 --- mariadb/scripts/start.sh | 9 - mariadb/supervisord.conf | 146 ------- mariadb/test/run | 413 ++++++++++++++++++ 28 files changed, 1287 insertions(+), 299 deletions(-) create mode 100644 mariadb/root/etc/my.cnf create mode 100755 mariadb/root/usr/bin/cgroup-limits create mode 100755 mariadb/root/usr/bin/container-entrypoint create mode 120000 mariadb/root/usr/bin/mysqld-master create mode 120000 mariadb/root/usr/bin/mysqld-slave create mode 100755 mariadb/root/usr/bin/run-mysqld create mode 100755 mariadb/root/usr/bin/run-mysqld-master create mode 100755 mariadb/root/usr/bin/run-mysqld-slave create mode 100755 mariadb/root/usr/libexec/container-setup create mode 100644 mariadb/root/usr/share/container-scripts/mysql/common.sh create mode 100644 mariadb/root/usr/share/container-scripts/mysql/helpers.sh create mode 100644 mariadb/root/usr/share/container-scripts/mysql/my-base.cnf.template create mode 100644 mariadb/root/usr/share/container-scripts/mysql/my-master.cnf.template create mode 100644 mariadb/root/usr/share/container-scripts/mysql/my-paas.cnf.template create mode 100644 mariadb/root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template create mode 100644 mariadb/root/usr/share/container-scripts/mysql/my-slave.cnf.template create mode 100644 mariadb/root/usr/share/container-scripts/mysql/my-tuning.cnf.template create mode 100644 mariadb/root/usr/share/container-scripts/mysql/passwd-change.sh create mode 100644 mariadb/root/usr/share/container-scripts/mysql/post-init.sh create mode 100644 mariadb/root/usr/share/container-scripts/mysql/scl_enable create mode 100644 mariadb/root/usr/share/container-scripts/mysql/validate-replication-variables.sh create mode 100644 mariadb/root/usr/share/container-scripts/mysql/validate-variables.sh delete mode 100644 mariadb/scripts/config_mariadb.sh delete mode 100644 mariadb/scripts/start.sh delete mode 100644 mariadb/supervisord.conf create mode 100755 mariadb/test/run diff --git a/mariadb/Dockerfile b/mariadb/Dockerfile index e5841123..09733564 100644 --- a/mariadb/Dockerfile +++ b/mariadb/Dockerfile @@ -1,13 +1,55 @@ FROM fedora MAINTAINER http://fedoraproject.org/wiki/Cloud -RUN dnf -y update && dnf clean all -RUN dnf -y install mariadb-server pwgen psmisc net-tools hostname && \ - dnf clean all -ADD scripts /scripts -RUN chmod 755 /scripts/* +# MariaDB image for OpenShift. +# +# Volumes: +# * /var/lib/mysql/data - Datastore for MariaDB +# Environment: +# * $MYSQL_USER - Database user name +# * $MYSQL_PASSWORD - User's password +# * $MYSQL_DATABASE - Name of the database to create +# * $MYSQL_ROOT_PASSWORD (Optional) - Password for the 'root' MySQL account + +ENV MYSQL_VERSION=10.0 \ + HOME=/var/lib/mysql + +LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ + io.k8s.description="MariaDB is a multi-user, multi-threaded SQL database server" \ + io.k8s.display-name="MariaDB 10.0" \ + io.openshift.expose-services="3306:mysql" \ + io.openshift.tags="database,mysql,mariadb,mariadb100" -VOLUME ["/var/lib/mysql", "/var/log/mysql"] EXPOSE 3306 -CMD ["/bin/bash", "/scripts/start.sh"] +# This image must forever use UID 27 for mysql user so our volumes are +# safe in the future. This should *never* change, the last test is there +# to make sure of that. +# https://git.fedorahosted.org/cgit/setup.git/tree/uidgid +# policycoreutils installed for restorecon called in container-setup script +RUN INSTALL_PKGS="rsync tar gettext hostname bind-utils policycoreutils mariadb-server" && \ + dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + dnf clean all && \ + mkdir -p /var/lib/mysql/data && chown -R mysql.0 /var/lib/mysql && \ + rpm -q --qf '%{version}' mariadb-server | grep -e '10\.0\.' && \ + test "$(id mysql)" = "uid=27(mysql) gid=27(mysql) groups=27(mysql)" + +# Get prefix path and path to scripts rather than hard-code them in scripts +ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/mysql \ + MYSQL_PREFIX=/usr + +ADD root / + +# this is needed due to issues with squash +# when this directory gets rm'd by the container-setup +# script. +RUN rm -rf /etc/my.cnf.d/* +RUN /usr/libexec/container-setup + +VOLUME ["/var/lib/mysql/data"] + +USER 27 + +ENTRYPOINT ["container-entrypoint"] +CMD ["run-mysqld"] diff --git a/mariadb/README.md b/mariadb/README.md index 24e29f12..9b1a7f7f 100644 --- a/mariadb/README.md +++ b/mariadb/README.md @@ -1,88 +1,131 @@ -dockerfiles-fedora-mariadb -========================== - -Based on scollier's mysql dockerfile. - -This repo contains a recipe for making a Docker container for mariadb -on Fedora. - -Setup ------ - -Check your Docker version - - # docker version - -Perform the build - - # docker build --rm -t /mariadb . - -Check the image out. - - # docker images - -Launching MariaDB ------------------ - -### Stand-alone database: ### - - # docker run --name=mariadb -d -p 3306:3306 /mariadb - -This will create the system tables and a database named 'db', with user 'dbuser' and a generated password. To find out what the password is, check the logs: - - # docker logs mariadb | grep -E '^USER|^PASS' - -### Adjustable configuration ### - -Create a data volume container: - - # docker run --name=mariadb-data -v /var/lib/mysql \ - --entrypoint /bin/echo /mariadb "MariaDB data volume" - -Now create the persistent container, using the data volume container for storage: - - # docker run --name=mariadb --volumes-from=mariadb-data \ - -p 3306:3306 -d /mariadb - -The container will not re-initialise an already-initialised data volume. - -Using your MariaDB container ----------------------------- - -Connecting to mariadb: - - # mysql --protocol=tcp db -udbuser -p - -Use the password indicated in the 'docker logs' output. - -Create a sample table: - - \> CREATE TABLE test (name VARCHAR(10), owner VARCHAR(10), - -> species VARCHAR(10), birth DATE, death DATE); - -Linking with another container ------------------------------- - -To arrange for linking with another container, set the USER, PASS, and NAME environment variables when creating the mariadb container. You don't need to expose any ports, as they are available to other containers automatically: - - # docker run --name=mariadb --volumes-from=mariadb-data \ - -e USER=user -e PASS=mypassword -e NAME=mydb \ - -d /mariadb - -This will create a database named 'mydb', and a user 'user' with the specified password. To link another container to this one, use the --link option to 'docker run': - - # docker run --link=mariadb:db -d /mydbapp - -As we've set the alias for the linked mariadb container to 'db', the 'mydbapp' container will have environment variables set to give it the information it needs: - - - DB_PORT will specify the protocol, host, and port - - DB_ENV_NAME will be 'mydb' - - DB_ENV_USER will be 'user' - - DB_ENV_PASS will be 'mypassword' - -Using mariadb as a client to an existing mariadb container ----------------------------------------------------------- - -To run a query against an existing container, using the client from this container image, create a new container linked to the existing one: - - # docker run --rm --link=mariadb:db -i -t /mariadb sh -c 'mysql -h $DB_PORT_3306_TCP_ADDR -P $DB_PORT_3306_TCP_PORT -u$DB_ENV_USER -p$DB_ENV_PASS' +MariaDB Docker image +==================== + +This container image includes MariaDB server 10.0 for OpenShift and general usage. + +Environment variables and volumes +---------------------------------- + +The image recognizes the following environment variables that you can set during +initialization by passing `-e VAR=VALUE` to the Docker run command. + +| Variable name | Description | +| :--------------------- | ----------------------------------------- | +| `MYSQL_USER` | User name for MySQL account to be created | +| `MYSQL_PASSWORD` | Password for the user account | +| `MYSQL_DATABASE` | Database name | +| `MYSQL_ROOT_PASSWORD` | Password for the root user (optional) | + +The following environment variables influence the MySQL configuration file. They are all optional. + +| Variable name | Description | Default +| :------------------------------ | ----------------------------------------------------------------- | ------------------------------- +| `MYSQL_LOWER_CASE_TABLE_NAMES` | Sets how the table names are stored and compared | 0 +| `MYSQL_MAX_CONNECTIONS` | The maximum permitted number of simultaneous client connections | 151 +| `MYSQL_MAX_ALLOWED_PACKET` | The maximum size of one packet or any generated/intermediate string | 200M +| `MYSQL_FT_MIN_WORD_LEN` | The minimum length of the word to be included in a FULLTEXT index | 4 +| `MYSQL_FT_MAX_WORD_LEN` | The maximum length of the word to be included in a FULLTEXT index | 20 +| `MYSQL_AIO` | Controls the `innodb_use_native_aio` setting value in case the native AIO is broken. See http://help.directadmin.com/item.php?id=529 | 1 +| `MYSQL_TABLE_OPEN_CACHE` | The number of open tables for all threads | 400 +| `MYSQL_KEY_BUFFER_SIZE` | The size of the buffer used for index blocks | 32M (or 10% of available memory) +| `MYSQL_SORT_BUFFER_SIZE` | The size of the buffer used for sorting | 256K +| `MYSQL_READ_BUFFER_SIZE` | The size of the buffer used for a sequential scan | 8M (or 5% of available memory) +| `MYSQL_INNODB_BUFFER_POOL_SIZE`| The size of the buffer pool where InnoDB caches table and index data | 32M (or 50% of available memory) +| `MYSQL_INNODB_LOG_FILE_SIZE` | The size of each log file in a log group | 8M (or 15% of available available) +| `MYSQL_INNODB_LOG_BUFFER_SIZE` | The size of the buffer that InnoDB uses to write to the log files on disk | 8M (or 15% of available memory) +| `MYSQL_DEFAULTS_FILE` | Point to an alternative configuration file | /etc/my.cnf +| `MYSQL_BINLOG_FORMAT` | Set sets the binlog format, supported values are `row` and `statement` | statement + +You can also set the following mount points by passing the `-v /host:/container` flag to Docker. + +| Volume mount point | Description | +| :----------------------- | -------------------- | +| `/var/lib/mysql/data` | MySQL data directory | + +**Notice: When mouting a directory from the host into the container, ensure that the mounted +directory has the appropriate permissions and that the owner and group of the directory +matches the user UID which is running inside the container.** + +Usage +--------------------------------- + +For this, we will assume that you are using the `fedora/mariadb` image. +If you want to set only the mandatory environment variables and not store +the database in a host directory, execute the following command: + +``` +$ docker run -d --name mariadb_database -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -p 3306:3306 fedora/mariadb +``` + +This will create a container named `mariadb_database` running MySQL with database +`db` and user with credentials `user:pass`. Port 3306 will be exposed and mapped +to the host. If you want your database to be persistent across container executions, +also add a `-v /host/db/path:/var/lib/mysql/data` argument. This will be the MySQL +data directory. + +If the database directory is not initialized, the entrypoint script will first +run [`mysql_install_db`](https://dev.mysql.com/doc/refman/5.6/en/mysql-install-db.html) +and setup necessary database users and passwords. After the database is initialized, +or if it was already present, `mysqld` is executed and will run as PID 1. You can + stop the detached container by running `docker stop mariadb_database`. + + +MariaDB auto-tuning +------------------- + +When the MySQL image is run with the `--memory` parameter set and you didn't +specify value for some parameters, their values will be automatically +calculated based on the available memory. + +| Variable name | Configuration parameter | Relative value +| :-------------------------------| ------------------------- | -------------- +| `MYSQL_KEY_BUFFER_SIZE` | `key_buffer_size` | 10% +| `MYSQL_READ_BUFFER_SIZE` | `read_buffer_size` | 5% +| `MYSQL_INNODB_BUFFER_POOL_SIZE` | `innodb_buffer_pool_size` | 50% +| `MYSQL_INNODB_LOG_FILE_SIZE` | `innodb_log_file_size` | 15% +| `MYSQL_INNODB_LOG_BUFFER_SIZE` | `innodb_log_buffer_size` | 15% + + +MySQL root user +--------------------------------- +The root user has no password set by default, only allowing local connections. +You can set it by setting the `MYSQL_ROOT_PASSWORD` environment variable. This +will allow you to login to the root account remotely. Local connections will +still not require a password. + +To disable remote root access, simply unset `MYSQL_ROOT_PASSWORD` and restart +the container. + + +Changing passwords +------------------ + +Since passwords are part of the image configuration, the only supported method +to change passwords for the database user (`MYSQL_USER`) and root user is by +changing the environment variables `MYSQL_PASSWORD` and `MYSQL_ROOT_PASSWORD`, +respectively. + +Changing database passwords through SQL statements or any way other than through +the environment variables aforementioned will cause a mismatch between the +values stored in the variables and the actual passwords. Whenever a database +container starts it will reset the passwords to the values stored in the +environment variables. + +Default my.cnf file +------------------- +With environment variables we are able to customize a lot of different parameters +or configurations for the mysql bootstrap configurations. If you'd prefer to use +your own configuration file, you can override the `MYSQL_DEFAULTS_FILE` env +variable with the full path of the file you wish to use. For example, the default +location is `/etc/my.cnf` but you can change it to `/etc/mysql/my.cnf` by setting + `MYSQL_DEFAULTS_FILE=/etc/mysql/my.cnf` + +Changing the replication binlog_format +-------------------------------------- +Some applications may wish to use `row` binlog_formats (for example, those built + with change-data-capture in mind). The default replication/binlog format is + `statement` but to change it you can set the `MYSQL_BINLOG_FORMAT` environment + variable. For example `MYSQL_BINLOG_FORMAT=row`. Now when you run the database + with `master` replication turned on (ie, set the Docker/container `cmd` to be +`run-mysqld-master`) the binlog will emit the actual data for the rows that change +as opposed to the statements (ie, DML like insert...) that caused the change. diff --git a/mariadb/root/etc/my.cnf b/mariadb/root/etc/my.cnf new file mode 100644 index 00000000..86ed688b --- /dev/null +++ b/mariadb/root/etc/my.cnf @@ -0,0 +1,10 @@ +[mysqld] + +# Disabling symbolic-links is recommended to prevent assorted security risks +symbolic-links = 0 + +# http://www.percona.com/blog/2008/05/31/dns-achilles-heel-mysql-installation/ +skip_name_resolve + +!includedir /etc/my.cnf.d + diff --git a/mariadb/root/usr/bin/cgroup-limits b/mariadb/root/usr/bin/cgroup-limits new file mode 100755 index 00000000..f50bbbb7 --- /dev/null +++ b/mariadb/root/usr/bin/cgroup-limits @@ -0,0 +1,92 @@ +#!/usr/bin/python3 + +""" +Script for parsing cgroup information + +This script will read some limits from the cgroup system and parse +them, printing out "VARIABLE=VALUE" on each line for every limit that is +successfully read. Output of this script can be directly fed into +bash's export command. Recommended usage from a bash script: + + set -o errexit + export_vars=$(cgroup-limits) ; export $export_vars + +Variables currently supported: + MAX_MEMORY_LIMIT_IN_BYTES + Maximum possible limit MEMORY_LIMIT_IN_BYTES can have. This is + currently constant value of 9223372036854775807. + MEMORY_LIMIT_IN_BYTES + Maximum amount of user memory in bytes. If this value is set + to the same value as MAX_MEMORY_LIMIT_IN_BYTES, it means that + there is no limit set. The value is taken from + /sys/fs/cgroup/memory/memory.limit_in_bytes + NUMBER_OF_CORES + Number of detected CPU cores that can be used. This value is + calculated from /sys/fs/cgroup/cpuset/cpuset.cpus + NO_MEMORY_LIMIT + Set to "true" if MEMORY_LIMIT_IN_BYTES is so high that the caller + can act as if no memory limit was set. Undefined otherwise. +""" + +from __future__ import print_function +import sys + + +def _read_file(path): + try: + with open(path, 'r') as f: + return f.read().strip() + except IOError: + return None + + +def get_memory_limit(): + """ + Read memory limit, in bytes. + """ + + limit = _read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes') + if limit is None or not limit.isdigit(): + print("Warning: Can't detect memory limit from cgroups", + file=sys.stderr) + return None + return int(limit) + + +def get_number_of_cores(): + """ + Read number of CPU cores. + """ + + core_count = 0 + + line = _read_file('/sys/fs/cgroup/cpuset/cpuset.cpus') + if line is None: + print("Warning: Can't detect number of CPU cores from cgroups", + file=sys.stderr) + return None + + for group in line.split(','): + core_ids = list(map(int, group.split('-'))) + if len(core_ids) == 2: + core_count += core_ids[1] - core_ids[0] + 1 + else: + core_count += 1 + + return core_count + + +if __name__ == "__main__": + env_vars = { + "MAX_MEMORY_LIMIT_IN_BYTES": 9223372036854775807, + "MEMORY_LIMIT_IN_BYTES": get_memory_limit(), + "NUMBER_OF_CORES": get_number_of_cores() + } + + env_vars = {k: v for k, v in env_vars.items() if v is not None} + + if env_vars.get("MEMORY_LIMIT_IN_BYTES", 0) >= 92233720368547: + env_vars["NO_MEMORY_LIMIT"] = "true" + + for key, value in env_vars.items(): + print("{0}={1}".format(key, value)) diff --git a/mariadb/root/usr/bin/container-entrypoint b/mariadb/root/usr/bin/container-entrypoint new file mode 100755 index 00000000..9d8ad4d3 --- /dev/null +++ b/mariadb/root/usr/bin/container-entrypoint @@ -0,0 +1,2 @@ +#!/bin/bash +exec "$@" diff --git a/mariadb/root/usr/bin/mysqld-master b/mariadb/root/usr/bin/mysqld-master new file mode 120000 index 00000000..8a0786e1 --- /dev/null +++ b/mariadb/root/usr/bin/mysqld-master @@ -0,0 +1 @@ +run-mysqld-master \ No newline at end of file diff --git a/mariadb/root/usr/bin/mysqld-slave b/mariadb/root/usr/bin/mysqld-slave new file mode 120000 index 00000000..dc0f58b2 --- /dev/null +++ b/mariadb/root/usr/bin/mysqld-slave @@ -0,0 +1 @@ +run-mysqld-slave \ No newline at end of file diff --git a/mariadb/root/usr/bin/run-mysqld b/mariadb/root/usr/bin/run-mysqld new file mode 100755 index 00000000..cd899a73 --- /dev/null +++ b/mariadb/root/usr/bin/run-mysqld @@ -0,0 +1,35 @@ +#!/bin/bash + +export_vars=$(cgroup-limits); export $export_vars +source ${CONTAINER_SCRIPTS_PATH}/common.sh +set -eu + +[ -f ${CONTAINER_SCRIPTS_PATH}/validate-variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate-variables.sh + +# Process the MySQL configuration files +log_info 'Processing MySQL configuration files ...' +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf + +if [ ! -d "$MYSQL_DATADIR/mysql" ]; then + initialize_database "$@" +else + start_local_mysql "$@" +fi + +if [ -f ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh ]; then + log_info 'Setting passwords ...' + source ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh +fi +if [ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ]; then + log_info 'Sourcing post-init.sh ...' + source ${CONTAINER_SCRIPTS_PATH}/post-init.sh +fi + +# Restart the MySQL server with public IP bindings +shutdown_local_mysql +unset_env_vars +log_volume_info $MYSQL_DATADIR +log_info 'Running final exec -- Only MySQL server logs after this point' +exec ${MYSQL_PREFIX}/libexec/mysqld --defaults-file=$MYSQL_DEFAULTS_FILE "$@" 2>&1 diff --git a/mariadb/root/usr/bin/run-mysqld-master b/mariadb/root/usr/bin/run-mysqld-master new file mode 100755 index 00000000..054889e4 --- /dev/null +++ b/mariadb/root/usr/bin/run-mysqld-master @@ -0,0 +1,50 @@ +#!/bin/bash +# +# This is an entrypoint that runs the MySQL server in the 'master' mode. +# +export_vars=$(cgroup-limits); export $export_vars +source ${CONTAINER_SCRIPTS_PATH}/common.sh +set -eu + +export MYSQL_RUNNING_AS_MASTER=1 + +[ -f ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh +[ -f ${CONTAINER_SCRIPTS_PATH}/validate_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_variables.sh + +# The 'server-id' for master needs to be constant +export MYSQL_SERVER_ID=1 +log_info "The 'master' server-id is ${MYSQL_SERVER_ID}" + +# Process the MySQL configuration files +log_info 'Processing MySQL configuration files ...' +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-master.cnf.template > /etc/my.cnf.d/master.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf + +if [ ! -d "$MYSQL_DATADIR/mysql" ]; then + initialize_database "$@" +else + start_local_mysql "$@" +fi + +log_info 'Setting passwords ...' +[ -f ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh ] && source ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh + +# Setup the 'master' replication on the MySQL server +mysql $mysql_flags <&1 diff --git a/mariadb/root/usr/bin/run-mysqld-slave b/mariadb/root/usr/bin/run-mysqld-slave new file mode 100755 index 00000000..51acce5d --- /dev/null +++ b/mariadb/root/usr/bin/run-mysqld-slave @@ -0,0 +1,60 @@ +#!/bin/bash +# +# This is an entrypoint that runs the MySQL server in the 'slave' mode. +# +export_vars=$(cgroup-limits); export $export_vars +source ${CONTAINER_SCRIPTS_PATH}/common.sh +set -eu + +# Just run normal server if the data directory is already initialized +if [ -d "${MYSQL_DATADIR}/mysql" ]; then + exec /usr/bin/run-mysqld "$@" +fi + +export MYSQL_RUNNING_AS_SLAVE=1 + +[ -f ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh + +# Generate the unique 'server-id' for this master +export MYSQL_SERVER_ID=$(server_id) +log_info "The 'slave' server-id is ${MYSQL_SERVER_ID}" + +# Process the MySQL configuration files +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-slave.cnf.template > /etc/my.cnf.d/slave.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf + +# Initialize MySQL database and wait for the MySQL master to accept +# connections. +initialize_database "$@" +wait_for_mysql_master + +# Get binlog file and position from master +STATUS_INFO=$(mysql --host "$MYSQL_MASTER_SERVICE_NAME" "-u${MYSQL_MASTER_USER}" "-p${MYSQL_MASTER_PASSWORD}" replication -e 'SELECT gtid from replication limit 1\G') +GTID_VALUE=$(echo "$STATUS_INFO" | grep 'gtid:' | head -n 1 | sed -e 's/^\s*gtid: //') + +# checking STATUS_INFO here because empty GTID_VALUE is valid value +if [ -z "${STATUS_INFO}" ] ; then + echo "Could not read GTID value from master" + exit 1 +fi + +mysql $mysql_flags <&1 diff --git a/mariadb/root/usr/libexec/container-setup b/mariadb/root/usr/libexec/container-setup new file mode 100755 index 00000000..29c6ed25 --- /dev/null +++ b/mariadb/root/usr/libexec/container-setup @@ -0,0 +1,58 @@ +#!/bin/bash + +# This function returns all config files that daemon uses and their path +# includes /opt. It is used to get correct path to the config file. +mysql_get_config_files_scl() { + scl enable ${ENABLED_COLLECTIONS} -- my_print_defaults --help --verbose | \ + grep --after=1 '^Default options' | \ + tail -n 1 | \ + grep -o '[^ ]*opt[^ ]*my.cnf' +} + +# This function picks the main config file that deamon uses and we ship in rpm +mysql_get_correct_config() { + # we use the same config in non-SCL packages, not necessary to guess + [ -z "${ENABLED_COLLECTIONS}" ] && echo -n "/etc/my.cnf" && return + + # from all config files read by daemon, pick the first that exists + for f in `mysql_get_config_files_scl` ; do + [ -f "$f" ] && echo "$f" + done | head -n 1 +} + +export MYSQL_CONFIG_FILE=$(mysql_get_correct_config) + +[ -z "$MYSQL_CONFIG_FILE" ] && echo "MYSQL_CONFIG_FILE is empty" && exit 1 + +unset -f mysql_get_correct_config mysql_get_config_files_scl + +# we provide own config files for the container, so clean what rpm ships here +mkdir -p ${MYSQL_CONFIG_FILE}.d +rm -f ${MYSQL_CONFIG_FILE}.d/* + +# we may add options during service init, so we need to have this dir writable by daemon user +chown -R mysql:0 ${MYSQL_CONFIG_FILE}.d ${MYSQL_CONFIG_FILE} +restorecon -R ${MYSQL_CONFIG_FILE}.d ${MYSQL_CONFIG_FILE} + +# API of the container are standard paths /etc/my.cnf and /etc/my.cnf.d +# we already include own /etc/my.cnf for container, but for cases the +# actually used config file is not on standard path /etc/my.cnf, we +# need to move it to the location daemon expects it and create symlinks +if [ "$MYSQL_CONFIG_FILE" != "/etc/my.cnf" ] ; then + rm -rf /etc/my.cnf.d + mv /etc/my.cnf ${MYSQL_CONFIG_FILE} + ln -s ${MYSQL_CONFIG_FILE} /etc/my.cnf + ln -s ${MYSQL_CONFIG_FILE}.d /etc/my.cnf.d +fi + +# setup directory for data +mkdir -p /var/lib/mysql/data +chown -R mysql:0 /var/lib/mysql +restorecon -R /var/lib/mysql + +# Loosen permission bits for group to avoid problems running container with +# arbitrary UID +# When only specifying user, group is 0, that's why /var/lib/mysql must have +# owner mysql.0; that allows to avoid a+rwx for this dir +chmod g+w -R /var/lib/mysql ${MYSQL_CONFIG_FILE}.d + diff --git a/mariadb/root/usr/share/container-scripts/mysql/common.sh b/mariadb/root/usr/share/container-scripts/mysql/common.sh new file mode 100644 index 00000000..f85ad4fb --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/common.sh @@ -0,0 +1,162 @@ +#!/bin/bash + +source ${CONTAINER_SCRIPTS_PATH}/helpers.sh + +# Data directory where MySQL database files live. The data subdirectory is here +# because .bashrc and my.cnf both live in /var/lib/mysql/ and we don't want a +# volume to override it. +export MYSQL_DATADIR=/var/lib/mysql/data + +# Configuration settings. +export MYSQL_DEFAULTS_FILE=${MYSQL_DEFAULTS_FILE:-/etc/my.cnf} +export MYSQL_BINLOG_FORMAT=${MYSQL_BINLOG_FORMAT:-STATEMENT} +export MYSQL_LOWER_CASE_TABLE_NAMES=${MYSQL_LOWER_CASE_TABLE_NAMES:-0} +export MYSQL_MAX_CONNECTIONS=${MYSQL_MAX_CONNECTIONS:-151} +export MYSQL_FT_MIN_WORD_LEN=${MYSQL_FT_MIN_WORD_LEN:-4} +export MYSQL_FT_MAX_WORD_LEN=${MYSQL_FT_MAX_WORD_LEN:-20} +export MYSQL_AIO=${MYSQL_AIO:-1} +export MYSQL_MAX_ALLOWED_PACKET=${MYSQL_MAX_ALLOWED_PACKET:-200M} +export MYSQL_TABLE_OPEN_CACHE=${MYSQL_TABLE_OPEN_CACHE:-400} +export MYSQL_SORT_BUFFER_SIZE=${MYSQL_SORT_BUFFER_SIZE:-256K} + +if [ -n "${NO_MEMORY_LIMIT:-}" -o -z "${MEMORY_LIMIT_IN_BYTES:-}" ]; then + key_buffer_size='32M' + read_buffer_size='8M' + innodb_buffer_pool_size='32M' + innodb_log_file_size='8M' + innodb_log_buffer_size='8M' +else + key_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.1))")M" + read_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.05))")M" + innodb_buffer_pool_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.5))")M" + innodb_log_file_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.15))")M" + innodb_log_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.15))")M" +fi +export MYSQL_KEY_BUFFER_SIZE=${MYSQL_KEY_BUFFER_SIZE:-$key_buffer_size} +export MYSQL_READ_BUFFER_SIZE=${MYSQL_READ_BUFFER_SIZE:-$read_buffer_size} +export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-$innodb_buffer_pool_size} +export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-$innodb_log_file_size} +export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-$innodb_log_buffer_size} + +# Be paranoid and stricter than we should be. +# https://dev.mysql.com/doc/refman/en/identifiers.html +mysql_identifier_regex='^[a-zA-Z0-9_]+$' +mysql_password_regex='^[a-zA-Z0-9_~!@#$%^&*()-=<>,.?;:|]+$' + +# Variables that are used to connect to local mysql during initialization +mysql_flags="-u root --socket=/tmp/mysql.sock" +admin_flags="--defaults-file=$MYSQL_DEFAULTS_FILE $mysql_flags" + +# Make sure env variables don't propagate to mysqld process. +function unset_env_vars() { + log_info 'Cleaning up environment variables MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE and MYSQL_ROOT_PASSWORD ...' + unset MYSQL_USER MYSQL_PASSWORD MYSQL_DATABASE MYSQL_ROOT_PASSWORD +} + +# Poll until MySQL responds to our ping. +function wait_for_mysql() { + pid=$1 ; shift + + while [ true ]; do + if [ -d "/proc/$pid" ]; then + mysqladmin --socket=/tmp/mysql.sock ping &>/dev/null && log_info "MySQL started successfully" && return 0 + else + return 1 + fi + log_info "Waiting for MySQL to start ..." + sleep 1 + done +} + +# Start local MySQL server with a defaults file +function start_local_mysql() { + log_info 'Starting MySQL server with disabled networking ...' + ${MYSQL_PREFIX}/libexec/mysqld \ + --defaults-file=$MYSQL_DEFAULTS_FILE \ + --skip-networking --socket=/tmp/mysql.sock "$@" & + mysql_pid=$! + wait_for_mysql $mysql_pid +} + +# Shutdown mysql flushing privileges +function shutdown_local_mysql() { + log_info 'Shutting down MySQL ...' + mysqladmin $admin_flags flush-privileges shutdown +} + +# Initialize the MySQL database (create user accounts and the initial database) +function initialize_database() { + log_info 'Initializing database ...' + log_info 'Running mysql_install_db ...' + # Using --rpm since we need mysql_install_db behaves as in RPM + mysql_install_db --rpm --datadir=$MYSQL_DATADIR + start_local_mysql "$@" + + if [ -v MYSQL_RUNNING_AS_SLAVE ]; then + log_info 'Initialization finished' + return 0 + fi + + if [ -v MYSQL_RUNNING_AS_MASTER ]; then + # Save master status into a separate database. + STATUS_INFO=$(mysql $admin_flags -e 'SHOW MASTER STATUS\G') + BINLOG_POSITION=$(echo "$STATUS_INFO" | grep 'Position:' | head -n 1 | sed -e 's/^\s*Position: //') + BINLOG_FILE=$(echo "$STATUS_INFO" | grep 'File:' | head -n 1 | sed -e 's/^\s*File: //') + GTID_INFO=$(mysql $admin_flags -e "SELECT BINLOG_GTID_POS('$BINLOG_FILE', '$BINLOG_POSITION') AS gtid_value \G") + GTID_VALUE=$(echo "$GTID_INFO" | grep 'gtid_value:' | head -n 1 | sed -e 's/^\s*gtid_value: //') + + mysqladmin $admin_flags create replication + mysql $admin_flags </dev/null && log_info "MySQL master is ready" && return 0 + sleep 1 + done +} diff --git a/mariadb/root/usr/share/container-scripts/mysql/helpers.sh b/mariadb/root/usr/share/container-scripts/mysql/helpers.sh new file mode 100644 index 00000000..4e832fcd --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/helpers.sh @@ -0,0 +1,24 @@ +function log_info { + echo "---> `date +%T` $@" +} + +function log_and_run { + log_info "Running $@" + "$@" +} + +function log_volume_info { + CONTAINER_DEBUG=${CONTAINER_DEBUG:-} + if [[ "${CONTAINER_DEBUG,,}" != "true" ]]; then + return + fi + + log_info "Volume info for $@:" + set +e + log_and_run mount + while [ $# -gt 0 ]; do + log_and_run ls -alZ $1 + shift + done + set -e +} diff --git a/mariadb/root/usr/share/container-scripts/mysql/my-base.cnf.template b/mariadb/root/usr/share/container-scripts/mysql/my-base.cnf.template new file mode 100644 index 00000000..c654f7f1 --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/my-base.cnf.template @@ -0,0 +1,5 @@ +[mysqld] +datadir = ${MYSQL_DATADIR} +basedir = ${MYSQL_PREFIX} +plugin-dir = ${MYSQL_PREFIX}/lib64/mysql/plugin + diff --git a/mariadb/root/usr/share/container-scripts/mysql/my-master.cnf.template b/mariadb/root/usr/share/container-scripts/mysql/my-master.cnf.template new file mode 100644 index 00000000..f434885f --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/my-master.cnf.template @@ -0,0 +1,7 @@ +[mysqld] + +server-id = ${MYSQL_SERVER_ID} +log_bin = ${MYSQL_DATADIR}/mysql-bin.log +binlog_do_db = mysql +binlog_do_db = ${MYSQL_DATABASE} +binlog_format = ${MYSQL_BINLOG_FORMAT} diff --git a/mariadb/root/usr/share/container-scripts/mysql/my-paas.cnf.template b/mariadb/root/usr/share/container-scripts/mysql/my-paas.cnf.template new file mode 100644 index 00000000..11ddd1fc --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/my-paas.cnf.template @@ -0,0 +1,26 @@ +[mysqld] +# +# Settings configured by the user +# + +# Sets how the table names are stored and compared. Default: 0 +lower_case_table_names = ${MYSQL_LOWER_CASE_TABLE_NAMES} + +# The maximum permitted number of simultaneous client connections. Default: 151 +max_connections = ${MYSQL_MAX_CONNECTIONS} + +# The minimum/maximum lengths of the word to be included in a FULLTEXT index. Default: 4/20 +ft_min_word_len = ${MYSQL_FT_MIN_WORD_LEN} +ft_max_word_len = ${MYSQL_FT_MAX_WORD_LEN} + +# In case the native AIO is broken. Default: 1 +# See http://help.directadmin.com/item.php?id=529 +innodb_use_native_aio = ${MYSQL_AIO} + +[myisamchk] +# The minimum/maximum lengths of the word to be included in a FULLTEXT index. Default: 4/20 +# +# To ensure that myisamchk and the server use the same values for full-text +# parameters, we placed them in both sections. +ft_min_word_len = ${MYSQL_FT_MIN_WORD_LEN} +ft_max_word_len = ${MYSQL_FT_MAX_WORD_LEN} diff --git a/mariadb/root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template b/mariadb/root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template new file mode 100644 index 00000000..a74a74c6 --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template @@ -0,0 +1,4 @@ +[mysqld] + +log-slave-updates = ON + diff --git a/mariadb/root/usr/share/container-scripts/mysql/my-slave.cnf.template b/mariadb/root/usr/share/container-scripts/mysql/my-slave.cnf.template new file mode 100644 index 00000000..5bdf1095 --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/my-slave.cnf.template @@ -0,0 +1,7 @@ +[mysqld] + +server-id = ${MYSQL_SERVER_ID} +log_bin = ${MYSQL_DATADIR}/mysql-bin.log +relay-log = ${MYSQL_DATADIR}/mysql-relay-bin.log +binlog_do_db = mysql +binlog_do_db = ${MYSQL_DATABASE} diff --git a/mariadb/root/usr/share/container-scripts/mysql/my-tuning.cnf.template b/mariadb/root/usr/share/container-scripts/mysql/my-tuning.cnf.template new file mode 100644 index 00000000..e90b69ac --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/my-tuning.cnf.template @@ -0,0 +1,28 @@ +[mysqld] +key_buffer_size = ${MYSQL_KEY_BUFFER_SIZE} +max_allowed_packet = ${MYSQL_MAX_ALLOWED_PACKET} +table_open_cache = ${MYSQL_TABLE_OPEN_CACHE} +sort_buffer_size = ${MYSQL_SORT_BUFFER_SIZE} +read_buffer_size = ${MYSQL_READ_BUFFER_SIZE} +read_rnd_buffer_size = 256K +net_buffer_length = 2K +thread_stack = 256K +myisam_sort_buffer_size = 2M + +# It is recommended that innodb_buffer_pool_size is configured to 50 to 75 percent of system memory. +innodb_buffer_pool_size = ${MYSQL_INNODB_BUFFER_POOL_SIZE} +innodb_additional_mem_pool_size = 2M +# Set .._log_file_size to 25 % of buffer pool size +innodb_log_file_size = ${MYSQL_INNODB_LOG_FILE_SIZE} +innodb_log_buffer_size = ${MYSQL_INNODB_LOG_BUFFER_SIZE} + +[mysqldump] +quick +max_allowed_packet = 16M + +[mysql] +no-auto-rehash + +[myisamchk] +key_buffer_size = 8M +sort_buffer_size = 8M diff --git a/mariadb/root/usr/share/container-scripts/mysql/passwd-change.sh b/mariadb/root/usr/share/container-scripts/mysql/passwd-change.sh new file mode 100644 index 00000000..cd3f0a38 --- /dev/null +++ b/mariadb/root/usr/share/container-scripts/mysql/passwd-change.sh @@ -0,0 +1,25 @@ +# Set the password for MySQL user and root everytime this container is started. +# This allows to change the password by editing the deployment configuration. +if [[ -v MYSQL_USER && -v MYSQL_PASSWORD ]]; then + mysql $mysql_flags </dev/null + local exit_status + exit_status=$(docker inspect -f '{{.State.ExitCode}}' $CONTAINER) + if [ "$exit_status" != "0" ]; then + echo "Inspecting container $CONTAINER" + docker inspect $CONTAINER + echo "Dumping logs for $CONTAINER" + docker logs $CONTAINER + fi + docker rm -v $CONTAINER >/dev/null + rm $cidfile + echo "Done." + done + rmdir $CIDFILE_DIR +} +trap cleanup EXIT SIGINT + +function get_cid() { + local id="$1" ; shift || return 1 + echo $(cat "$CIDFILE_DIR/$id") +} + +function get_container_ip() { + local id="$1" ; shift + docker inspect --format='{{.NetworkSettings.IPAddress}}' $(get_cid "$id") +} + +function mysql_cmd() { + local container_ip="$1"; shift + local login="$1"; shift + local password="$1"; shift + docker run --rm "$IMAGE_NAME" mysql --host "$container_ip" -u"$login" -p"$password" "$@" db +} + +function test_connection() { + local name=$1 ; shift + local login=$1 ; shift + local password=$1 ; shift + local ip + ip=$(get_container_ip $name) + echo " Testing MySQL connection to $ip..." + local max_attempts=20 + local sleep_time=2 + local i + for i in $(seq $max_attempts); do + echo " Trying to connect..." + if mysql_cmd "$ip" "$login" "$password" <<< 'SELECT 1;'; then + echo " Success!" + return 0 + fi + sleep $sleep_time + done + echo " Giving up: Failed to connect. Logs:" + docker logs $(get_cid $name) + return 1 +} + +function test_mysql() { + local container_ip="$1" + local login="$2" + local password="$3" + + echo " Testing MySQL" + mysql_cmd "$container_ip" "$login" "$password" <<< 'CREATE TABLE tbl (col1 VARCHAR(20), col2 VARCHAR(20));' + mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo1", "bar1");' + mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo2", "bar2");' + mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo3", "bar3");' + mysql_cmd "$container_ip" "$login" "$password" <<< 'SELECT * FROM tbl;' + mysql_cmd "$container_ip" "$login" "$password" <<< 'DROP TABLE tbl;' + echo " Success!" +} + +function create_container() { + local name=$1 ; shift + cidfile="$CIDFILE_DIR/$name" + # create container with a cidfile in a directory for cleanup + local container_id + container_id="$(docker run ${DOCKER_ARGS:-} --cidfile $cidfile -d "$@" $IMAGE_NAME ${CONTAINER_ARGS:-})" + echo "Created container $container_id" +} + +function run_change_password_test() { + local tmpdir=$(mktemp -d) + mkdir "${tmpdir}/data" && chmod -R a+rwx "${tmpdir}" + + # Create MySQL container with persistent volume and set the initial password + create_container "testpass1" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z + test_connection testpass1 user foo + docker stop $(get_cid testpass1) >/dev/null + + # Create second container with changed password + create_container "testpass2" -e MYSQL_USER=user -e MYSQL_PASSWORD=bar \ + -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z + test_connection testpass2 user bar + + # The old password should not work anymore + if mysql_cmd "$(get_container_ip testpass2)" user foo -e 'SELECT 1;'; then + return 1 + fi +} + +function run_replication_test() { + local cluster_args="-e MYSQL_MASTER_USER=master -e MYSQL_MASTER_PASSWORD=master -e MYSQL_DATABASE=db" + local max_attempts=30 + + # Run the MySQL master + docker run $cluster_args -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_ROOT_PASSWORD=root \ + -e MYSQL_INNODB_BUFFER_POOL_SIZE=5M \ + -d --cidfile ${CIDFILE_DIR}/master.cid $IMAGE_NAME mysqld-master >/dev/null + local master_ip + master_ip=$(get_container_ip master.cid) + + # Run the MySQL slave + docker run $cluster_args -e MYSQL_MASTER_SERVICE_NAME=${master_ip} \ + -e MYSQL_INNODB_BUFFER_POOL_SIZE=5M \ + -d --cidfile ${CIDFILE_DIR}/slave.cid $IMAGE_NAME mysqld-slave >/dev/null + local slave_ip + slave_ip=$(get_container_ip slave.cid) + + # Now wait till the MASTER will see the SLAVE + local i + for i in $(seq $max_attempts); do + result="$(mysql_cmd "$master_ip" root root -e 'SHOW SLAVE HOSTS;' | grep "$slave_ip" || true)" + if [[ -n "${result}" ]]; then + echo "${slave_ip} successfully registered as SLAVE for ${master_ip}" + break + fi + if [[ "${i}" == "${max_attempts}" ]]; then + echo "The ${slave_ip} failed to register in MASTER" + echo "Dumping logs for $(get_cid slave.cid)" + docker logs $(get_cid slave.cid) + return 1 + fi + sleep 1 + done + + # do some real work to test replication in practice + mysql_cmd "$master_ip" root root -e "CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (24);" + + # read value from slave and check whether it is expectd + for i in $(seq $max_attempts); do + set +e + result="$(mysql_cmd "${slave_ip}" root root -e "select * from t1 \G" | grep -e ^a | grep 24)" + set -e + if [[ ! -z "${result}" ]]; then + echo "${slave_ip} successfully got value from MASTER ${master_ip}" + break + fi + if [[ "${i}" == "${max_attempts}" ]]; then + echo "The ${slave_ip} failed to see value added on MASTER" + echo "Dumping logs for $(get_cid slave.cid)" + docker logs $(get_cid slave.cid) + return 1 + fi + sleep 1 + done +} + +function assert_login_access() { + local container_ip=$1; shift + local USER=$1 ; shift + local PASS=$1 ; shift + local success=$1 ; shift + + if mysql_cmd "$container_ip" "$USER" "$PASS" <<< 'SELECT 1;' ; then + if $success ; then + echo " $USER($PASS) access granted as expected" + return + fi + else + if ! $success ; then + echo " $USER($PASS) access denied as expected" + return + fi + fi + echo " $USER($PASS) login assertion failed" + exit 1 +} + +function assert_local_access() { + local id="$1" ; shift + docker exec $(get_cid "$id") bash -c mysql <<< "SELECT 1;" +} + +# Make sure the invocation of docker run fails. +function assert_container_creation_fails() { + + # Time the docker run command. It should fail. If it doesn't fail, + # mysqld will keep running so we kill it with SIGKILL to make sure + # timeout returns a non-zero value. + local ret=0 + timeout -s 9 --preserve-status 60s docker run --rm "$@" $IMAGE_NAME >/dev/null || ret=$? + + # Timeout will exit with a high number. + if [ $ret -gt 30 ]; then + return 1 + fi +} + +function try_image_invalid_combinations() { + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_DATABASE=db "$@" + assert_container_creation_fails -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db "$@" +} + +function run_container_creation_tests() { + echo " Testing image entrypoint usage" + assert_container_creation_fails + try_image_invalid_combinations + try_image_invalid_combinations -e MYSQL_ROOT_PASSWORD=root_pass + + local VERY_LONG_DB_NAME="very_long_database_name_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass + assert_container_creation_fails -e MYSQL_USER=\$invalid -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=very_long_username -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD="\"" -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=\$invalid -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=$VERY_LONG_DB_NAME -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD="\"" + assert_container_creation_fails -e MYSQL_USER=root -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=pass + echo " Success!" +} + +function test_config_option() { + local container_name="$1" + local configuration="$2" + local option_name="$3" + local option_value="$4" + + if ! echo "$configuration" | grep -qx "$option_name[[:space:]]*=[[:space:]]*$option_value"; then + local configs="$(docker exec -t "$container_name" bash -c 'set +f; shopt -s nullglob; echo /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/* | paste -s')" + echo >&2 "FAIL: option '$option_name' should have value '$option_value', but it wasn't found in any of the configuration files ($configs):" + echo >&2 + echo >&2 "$configuration" + echo >&2 + return 1 + fi + + return 0 +} + +function run_configuration_tests() { + echo " Testing image configuration settings" + + local container_name=config_test + + create_container \ + "$container_name" \ + --name "$container_name" \ + --env MYSQL_USER=config_test_user \ + --env MYSQL_PASSWORD=config_test \ + --env MYSQL_DATABASE=db \ + --env MYSQL_LOWER_CASE_TABLE_NAMES=1 \ + --env MYSQL_MAX_CONNECTIONS=1337 \ + --env MYSQL_FT_MIN_WORD_LEN=8 \ + --env MYSQL_FT_MAX_WORD_LEN=15 \ + --env MYSQL_MAX_ALLOWED_PACKET=10M \ + --env MYSQL_TABLE_OPEN_CACHE=100 \ + --env MYSQL_SORT_BUFFER_SIZE=256K \ + --env MYSQL_KEY_BUFFER_SIZE=16M \ + --env MYSQL_READ_BUFFER_SIZE=16M \ + --env MYSQL_INNODB_BUFFER_POOL_SIZE=16M \ + --env MYSQL_INNODB_LOG_FILE_SIZE=4M \ + --env MYSQL_INNODB_LOG_BUFFER_SIZE=4M \ + --env WORKAROUND_DOCKER_BUG_14203= + # + + test_connection "$container_name" config_test_user config_test + + # TODO: this check is far from perfect and could be improved: + # - we should look for an option in the desired config, not in all of them + # - we should respect section of the config (now we have duplicated options from a different sections) + local configuration + configuration="$(docker exec -t "$container_name" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" + + test_config_option "$container_name" "$configuration" lower_case_table_names 1 + test_config_option "$container_name" "$configuration" max_connections 1337 + test_config_option "$container_name" "$configuration" ft_min_word_len 8 + test_config_option "$container_name" "$configuration" ft_max_word_len 15 + test_config_option "$container_name" "$configuration" max_allowed_packet 10M + test_config_option "$container_name" "$configuration" table_open_cache 100 + test_config_option "$container_name" "$configuration" sort_buffer_size 256K + test_config_option "$container_name" "$configuration" key_buffer_size 16M + test_config_option "$container_name" "$configuration" read_buffer_size 16M + test_config_option "$container_name" "$configuration" innodb_buffer_pool_size 16M + test_config_option "$container_name" "$configuration" innodb_log_file_size 4M + test_config_option "$container_name" "$configuration" innodb_log_buffer_size 4M + + docker stop "$container_name" >/dev/null + + echo " Success!" + echo " Testing image auto-calculated configuration settings" + + container_name=dynamic_config_test + + DOCKER_ARGS='--memory=256m' create_container \ + "$container_name" \ + --name "$container_name" \ + --env MYSQL_USER=config_test_user \ + --env MYSQL_PASSWORD=config_test \ + --env MYSQL_DATABASE=db + + test_connection "$container_name" config_test_user config_test + + configuration="$(docker exec -t "$container_name" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" + + test_config_option "$container_name" "$configuration" key_buffer_size 25M + test_config_option "$container_name" "$configuration" read_buffer_size 12M + test_config_option "$container_name" "$configuration" innodb_buffer_pool_size 128M + test_config_option "$container_name" "$configuration" innodb_log_file_size 38M + test_config_option "$container_name" "$configuration" innodb_log_buffer_size 38M + + docker stop "$container_name" >/dev/null + + echo " Success!" +} + +test_scl_usage() { + local name="$1" + local run_cmd="$2" + local expected="$3" + + echo " Testing the image SCL enable" + local out + out=$(docker run --rm ${IMAGE_NAME} /bin/bash -c "${run_cmd}") + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[/bin/bash -c "${run_cmd}"] Expected '${expected}', got '${out}'" + return 1 + fi + out=$(docker exec $(get_cid $name) /bin/bash -c "${run_cmd}" 2>&1) + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[exec /bin/bash -c "${run_cmd}"] Expected '${expected}', got '${out}'" + return 1 + fi + out=$(docker exec $(get_cid $name) /bin/sh -ic "${run_cmd}" 2>&1) + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[exec /bin/sh -ic "${run_cmd}"] Expected '${expected}', got '${out}'" + return 1 + fi +} + +function run_tests() { + local name=$1 ; shift + envs="-e MYSQL_USER=$USER -e MYSQL_PASSWORD=$PASS -e MYSQL_DATABASE=db" + if [ -v ROOT_PASS ]; then + envs="$envs -e MYSQL_ROOT_PASSWORD=$ROOT_PASS" + fi + create_container $name $envs + test_connection "$name" "$USER" "$PASS" + echo " Testing scl usage" + test_scl_usage $name 'mysql --version' '10.0' + echo " Testing login accesses" + local container_ip + container_ip=$(get_container_ip $name) + assert_login_access "$container_ip" "$USER" "$PASS" true + assert_login_access "$container_ip" "$USER" "${PASS}_foo" false + if [ -v ROOT_PASS ]; then + assert_login_access "$container_ip" root "$ROOT_PASS" true + assert_login_access "$container_ip" root "${ROOT_PASS}_foo" false + else + assert_login_access "$container_ip" root 'foo' false + assert_login_access "$container_ip" root '' false + fi + assert_local_access "$name" + echo " Success!" + test_mysql "$container_ip" "$USER" "$PASS" +} + +# Tests. + +run_container_creation_tests + +run_configuration_tests + +# Set lower buffer pool size to avoid running out of memory. +export CONTAINER_ARGS="run-mysqld --innodb_buffer_pool_size=5242880" + +# Normal tests +USER=user PASS=pass run_tests no_root +USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root +# Test with arbitrary uid for the container +DOCKER_ARGS="-u 12345" USER=user PASS=pass run_tests no_root_altuid +DOCKER_ARGS="-u 12345" USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root_altuid + +# Test the password change +run_change_password_test + +# Replication tests +run_replication_test