diff --git a/.github/workflows/balena.yml b/.github/workflows/balena.yml index 4cd8954..ae94396 100644 --- a/.github/workflows/balena.yml +++ b/.github/workflows/balena.yml @@ -8,6 +8,11 @@ on: branches: - master +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + # cancel jobs in progress for updated PRs, but not merge or tag events + cancel-in-progress: ${{ github.event.action == 'synchronize' }} + env: ENVIRONMENT: balena-cloud.com BLOCK_PREFIX: belodetek/unzoner diff --git a/.gitsecret/paths/mapping.cfg b/.gitsecret/paths/mapping.cfg index fa32711..5284212 100644 --- a/.gitsecret/paths/mapping.cfg +++ b/.gitsecret/paths/mapping.cfg @@ -2,6 +2,6 @@ unzoner/openvpn/client.key:1f2e908d2eac66568e4d4cc2c37967684c0da9cb32a24c970ee82 unzoner/openvpn/server.key:0817ed6b2ae19fdf1ddab8b507d70e7cbea29fa00674687eb845514539ad2c64 unzoner/openvpn/ta.key:61a616ec66587848c1a43cc92aa21ad0305e804581f9c2a437929713c4f4edb6 unzoner/id_rsa:7efaea18eedb4a1b9f9298e0f3d92ceb3f2adc4c29a5e818009a7d88f847296f -unzoner/.balena/secrets/env:b581220016ab9a0e9bb870baa31c1407caac3ba674ad785d3bffc9cb407f756a +unzoner/.balena/secrets/env:57af61f0f7fdb6afa4d15d7399358c00957b4358de192eb0d7b048f575e345a1 unzoner/mgmt.ovpn:b81b41c4cf71cdd490feecadf4bb35c31ed8396ed0fdd61808d4efd73c3031de unzoner/openvpn/dh2048.pem:7afe4bea0e57bc35bcd7cdd9e393692c43a88facc64c309a176dc69a66d5a029 diff --git a/unzoner/.balena/balena.yml b/unzoner/.balena/balena.yml index 6a16100..7f9adba 100644 --- a/unzoner/.balena/balena.yml +++ b/unzoner/.balena/balena.yml @@ -6,23 +6,22 @@ build-variables: # https://en.wikipedia.org/wiki/Bird_Internet_routing_daemon - BUILD_BIRD=0 # https://github.com/openssl/openssl - - BUILD_OPENSSL_VERSION=3.0.5 - - BUILD_OPENSSL=0 + # Setting up openssh-server (1:8.9p1-3ubuntu0.4) ... + # OpenSSL version mismatch. Built against 30000020, you have 30200000 + - BUILD_OPENSSL_VERSION=3.0.12 + - BUILD_OPENSSL=1 # https://gcc.gnu.org/onlinedocs/gcc/AArch64-Options.html - AARCH64_OPTIMISE_FLAGS=-O3 -mtune=cortex-a72 -march=armv8-a # https://gcc.gnu.org/onlinedocs/gcc/ARM-Options.html - ARM_OPTIMISE_FLAGS=-O3 -mtune=cortex-a53 -mcpu=cortex-a53+crypto -mfpu=crypto-neon-fp-armv8 # https://github.com/OpenVPN/openvpn - - BUILD_OPENVPN_VERSION=2.5.7 - - BUILD_OPENVPN=0 + - BUILD_OPENVPN_VERSION=2.6.8 + - BUILD_OPENVPN=1 # FIXME: https://sources.debian.org/patches/sniproxy/0.6.0-2/ - - BUILD_SNIPROXY_VERSION=0.6.0 - - BUILD_SNIPROXY=0 - # project abandoned http://wanproxy.org/) - - BUILD_WANPROXY_VERSION=0.8.0 - - BUILD_WANPROXY=0 + - BUILD_SNIPROXY_VERSION=0.6.1 + - BUILD_SNIPROXY=1 # https://nuitka.net/ - - COMPILE_CODE=0 + - COMPILE_CODE=1 # (e.g.) dig +short us.{{ DNS_SUB_DOMAIN }}.{{ DNS_DOMAIN }} - DNS_SUB_DOMAIN=blackbox diff --git a/unzoner/.balena/secrets/env.secret b/unzoner/.balena/secrets/env.secret index adc913c..3292a17 100644 Binary files a/unzoner/.balena/secrets/env.secret and b/unzoner/.balena/secrets/env.secret differ diff --git a/unzoner/Dockerfile.template b/unzoner/Dockerfile.template index c71497e..56e7b6c 100644 --- a/unzoner/Dockerfile.template +++ b/unzoner/Dockerfile.template @@ -14,44 +14,48 @@ ARG BUILD_OPENVPN ARG BUILD_OPENVPN_VERSION ARG BUILD_SNIPROXY ARG BUILD_SNIPROXY_VERSION -ARG BUILD_WANPROXY -ARG BUILD_WANPROXY_VERSION ARG COMPILE_CODE ARG DNS_SUB_DOMAIN ENV DEBIAN_FRONTEND noninteractive RUN install_packages \ - automake \ - bison \ - ca-certificates \ - cmake \ - curl \ - fakeroot \ - flex \ - gawk \ - gettext \ - libev-dev \ - liblz4-dev \ - liblzo2-dev \ - libpam-dev \ - libssl-dev \ - libtool \ - libudns-dev \ - openssl \ - python3-venv + automake \ + bison \ + ca-certificates \ + cmake \ + curl \ + fakeroot \ + flex \ + gawk \ + gettext \ + libcap-ng-dev \ + libev-dev \ + liblz4-dev \ + liblzo2-dev \ + libnl-genl-3-dev \ + libpam0g-dev \ + libssl-dev \ + libtool \ + libudns-dev \ + openssl \ + python3-venv WORKDIR /data RUN useradd openssl && (passwd -d openssl || true) +ENV LD_LIBRARY_PATH="/usr/local/ssl/lib64:$LD_LIBRARY_PATH" + +RUN update-ca-certificates + RUN if [ "$BUILD_OPENSSL" = '1' ]; then \ - set -x; git clone --single-branch --branch openssl-$BUILD_OPENSSL_VERSION https://github.com/openssl/openssl \ - && cd openssl \ - && chown -hR openssl:openssl . \ - && if [ '%%BALENA_ARCH%%' = 'armv7hf' ]; then ./config --prefix=/usr/local/ssl shared threads no-async $ARM_OPTIMISE_FLAGS; \ - elif [ '%%BALENA_ARCH%%' = 'aarch64' ]; then ./config --prefix=/usr/local/ssl shared threads no-async $AARCH64_OPTIMISE_FLAGS; \ - else ./config --prefix=/usr/local/ssl shared threads no-async; fi && make -j $(nproc); fi + set -x; git clone --depth 1 --branch openssl-$BUILD_OPENSSL_VERSION https://github.com/openssl/openssl \ + && cd openssl \ + && chown -hR openssl:openssl . \ + && if [ '%%BALENA_ARCH%%' = 'armv7hf' ]; then ./config --prefix=/usr/local/ssl shared threads no-async $ARM_OPTIMISE_FLAGS; \ + elif [ '%%BALENA_ARCH%%' = 'aarch64' ]; then ./config --prefix=/usr/local/ssl shared threads no-async $AARCH64_OPTIMISE_FLAGS; \ + else ./config --prefix=/usr/local/ssl shared threads no-async; fi && make -j $(nproc); fi USER openssl @@ -62,39 +66,33 @@ USER root RUN (deluser --remove-home openssl && delgroup openssl) || true RUN if [ "$BUILD_OPENSSL" = '1' ]; then \ - set -x; cd openssl && id && make install_sw \ - && /usr/local/ssl/bin/openssl version && ldd /usr/local/ssl/bin/openssl; fi + set -x; cd openssl && id && make install_sw && ldconfig /usr/local/ssl/lib64 \ + && /usr/local/ssl/bin/openssl version && ldd /usr/local/ssl/bin/openssl; fi # https://stackoverflow.com/a/39006247/1559300 RUN if [ "$BUILD_OPENVPN" = '1' ]; then \ - set -x; wget -q https://swupdate.openvpn.org/community/releases/openvpn-$BUILD_OPENVPN_VERSION.tar.gz \ - && tar -xvf openvpn-$BUILD_OPENVPN_VERSION.tar.gz && cd openvpn-$BUILD_OPENVPN_VERSION \ - && if [ "$BUILD_OPENSSL" = '1' ]; then CFLAGS='-I/usr/local/ssl/include -Wl,-rpath=/usr/local/ssl/lib -Wl,-rpath=/usr/local/ssl/lib64 -L/usr/local/ssl/lib -L/usr/local/ssl/lib64' ./configure; else ./configure; fi \ - && make -j $(nproc) && make check && make install \ - && /usr/local/sbin/openvpn --version \ - && ldd /usr/local/sbin/openvpn; fi + set -x; curl --retry 3 --silent --fail -o openvpn-$BUILD_OPENVPN_VERSION.tar.gz https://swupdate.openvpn.org/community/releases/openvpn-$BUILD_OPENVPN_VERSION.tar.gz \ + && tar -xvf openvpn-$BUILD_OPENVPN_VERSION.tar.gz && cd openvpn-$BUILD_OPENVPN_VERSION \ + && if [ "$BUILD_OPENSSL" = '1' ]; then CFLAGS='-I/usr/local/ssl/include -Wl,-rpath=/usr/local/ssl/lib -Wl,-rpath=/usr/local/ssl/lib64 -L/usr/local/ssl/lib -L/usr/local/ssl/lib64' ./configure; else ./configure; fi \ + && make -j $(nproc) && make check && make install \ + && /usr/local/sbin/openvpn --version \ + && ldd /usr/local/sbin/openvpn; fi RUN if [ "$BUILD_SNIPROXY" = '1' ]; then \ - git clone https://github.com/dlundquist/sniproxy.git \ - && cd sniproxy \ - && git checkout $BUILD_SNIPROXY_VERSION \ - && ./autogen.sh \ - && ./configure \ - && make install; fi + git clone https://github.com/dlundquist/sniproxy.git \ + && cd sniproxy \ + && git checkout $BUILD_SNIPROXY_VERSION \ + && ./autogen.sh \ + && ./configure \ + && make install; fi RUN if [ "$BUILD_BIRD" = '1' ]; then \ - wget -q https://bird.network.cz/download/bird-$BUILD_BIRD_VERSION.tar.gz \ - && tar -xvf bird-$BUILD_BIRD_VERSION.tar.gz && cd bird-$BUILD_BIRD_VERSION \ - && mkdir -p tools/ \ - && wget -qO tools/config.guess 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' \ - && wget -qO tools/config.sub 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' \ - && autoreconf && ./configure && make -j $(nproc) && make install; fi - -RUN if [ "$BUILD_WANPROXY" = '1' ]; then \ - wget -q http://wanproxy.org/releases/wanproxy-$BUILD_WANPROXY_VERSION.tar.gz \ - && tar zxf wanproxy-$BUILD_WANPROXY_VERSION.tar.gz \ - && cd wanproxy-$BUILD_WANPROXY_VERSION/programs/wanproxy \ - && make && cp wanproxy /usr/local/sbin; fi + set -x; wget -qO bird-$BUILD_BIRD_VERSION.tar.gz https://bird.network.cz/download/bird-$BUILD_BIRD_VERSION.tar.gz \ + && tar -xvf bird-$BUILD_BIRD_VERSION.tar.gz && cd bird-$BUILD_BIRD_VERSION \ + && mkdir -p tools/ \ + && wget -qO tools/config.guess 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' \ + && wget -qO tools/config.sub 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' \ + && autoreconf && ./configure && make -j $(nproc) && make install; fi WORKDIR /root @@ -107,7 +105,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY requirements.txt /requirements.txt RUN pip3 install --upgrade pip setuptools wheel \ - && pip3 install --upgrade -r /requirements.txt + && pip3 install --upgrade -r /requirements.txt COPY src/tests/requirements.txt /_requirements.txt @@ -129,40 +127,33 @@ WORKDIR /root/build RUN set -a && . /run/secrets/env && src/tests/run RUN if [ "$COMPILE_CODE" = '1' ]; then \ - install_packages ccache \ - && pip3 install --upgrade \ - nuitka \ - ordered-set \ - patchelf \ - && mkdir -p app && cd app \ - && nuitka3 \ - --remove-output \ - --assume-yes-for-downloads \ - --output-dir=$(uname -m) \ - --plugin-enable=pylint-warnings \ - --standalone application.py \ - && cd "$(uname -m)/application.dist" && ln -s ../../templates/ && cd ../.. \ - && cd .. && mkdir -p src && cd src \ - && for src in log gen_hash as_prefixes main auth client; do \ - nuitka3 \ - --remove-output \ - --assume-yes-for-downloads \ - --output-dir=$(uname -m) \ - --plugin-enable=pylint-warnings \ - --standalone $src.py; done; fi + install_packages ccache \ + && pip3 install --upgrade \ + nuitka \ + ordered-set \ + patchelf \ + && mkdir -p app && cd app \ + && nuitka3 \ + --remove-output \ + --assume-yes-for-downloads \ + --output-dir=$(uname -m) \ + --plugin-enable=pylint-warnings \ + --standalone application.py \ + && cd "$(uname -m)/application.dist" && ln -s ../../templates/ && cd ../.. \ + && cd .. && mkdir -p src && cd src \ + && for src in log gen_hash as_prefixes main auth client; do \ + nuitka3 \ + --remove-output \ + --assume-yes-for-downloads \ + --output-dir=$(uname -m) \ + --plugin-enable=pylint-warnings \ + --standalone $src.py; done; fi # creates encrypted app.tgz.enc bundle RUN set -a && . /run/secrets/env && utils/encrypt.sh WORKDIR /opt -RUN set -a \ - && . /run/secrets/env \ - && mkdir -p /usr/share/GeoIP/ \ - && wget -q "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=$MAXMIND_LICENSE_KEY&suffix=tar.gz" -O - | gunzip -d - > /usr/share/GeoIP/GeoIP.dat \ - && wget -q "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=$MAXMIND_LICENSE_KEY&suffix=tar.gz" -O - | gunzip -d - > /usr/share/GeoIP/GeoLiteCity.dat \ - && wget -q "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&license_key=$MAXMIND_LICENSE_KEY&suffix=tar.gz" -O - | gunzip -d - > /usr/share/GeoIP/GeoIPASNum.dat - COPY systemd/* ./systemd/ @@ -177,6 +168,8 @@ ENV VIRTUAL_ENV /root/venv3 ENV PATH "$VIRTUAL_ENV/bin:$PATH" +ENV LD_LIBRARY_PATH="/usr/local/ssl/lib64:$LD_LIBRARY_PATH" + ENV DEBIAN_FRONTEND noninteractive ENV PYTHONUNBUFFERED 1 @@ -191,8 +184,6 @@ COPY --from=build /usr/local/ /usr/local/ COPY --from=build /root/venv3/ /root/venv3/ -COPY --from=build /usr/share/GeoIP/ /usr/share/GeoIP/ - COPY --from=build /opt/systemd/entry.sh /usr/bin/entry.sh COPY --from=build /opt/systemd/balena.service /etc/systemd/system/balena.service @@ -201,75 +192,78 @@ RUN [ -f /usr/local/sbin/openvpn ] || install_packages openvpn RUN [ -f /usr/local/sbin/sniproxy ] || install_packages sniproxy -RUN [ -f /usr/local/ssl/bin/openssl ] && (rm -f /usr/local/bin/openssl && ln -s /usr/local/ssl/bin/openssl /usr/local/bin/openssl) \ - || install_packages openssl +RUN [ -f /usr/local/ssl/bin/openssl ] \ + && (rm -f /usr/local/bin/openssl && ln -s /usr/local/ssl/bin/openssl /usr/local/bin/openssl) \ + || install_packages openssl RUN which bird || install_packages bird RUN install_packages \ - bash \ - bridge-utils \ - ca-certificates \ - coreutils \ - cryptsetup \ - curl \ - dbus \ - dnsmasq \ - dnsutils \ - e2fsprogs \ - fdisk \ - findutils \ - gawk \ - geoip-bin \ - geoip-database \ - gettext \ - git \ - grep \ - haveged \ - hdparm \ - hostapd \ - iftop \ - iperf \ - iproute2 \ - ipset \ - iptables \ - iputils-ping \ - iputils-tracepath \ - iw \ - jq \ - kmod \ - libev4 \ - libevent-2.1 \ - liblz4-1 \ - liblzo2-2 \ - libpcre3 \ - libtool \ - libudns0 \ - linux-firmware \ - lsof \ - miniupnpc \ - mtr \ - net-tools \ - netcat-openbsd \ - nmap \ - openntpd \ - openssh-server \ - procps \ - psmisc \ - sipcalc\ - socat \ - stunnel \ - sysstat \ - systemd-sysv \ - tcpdump \ - telnet \ - udev \ - usbutils \ - vim \ - wget \ - whois \ - wireless-tools \ - zlib1g + bash \ + bridge-utils \ + ca-certificates \ + coreutils \ + cryptsetup \ + curl \ + dbus \ + dnsmasq \ + dnsutils \ + e2fsprogs \ + fdisk \ + findutils \ + gawk \ + gettext \ + git \ + grep \ + haveged \ + hdparm \ + hostapd \ + iftop \ + iperf \ + iproute2 \ + ipset \ + iptables \ + iputils-ping \ + iputils-tracepath \ + iw \ + jq \ + kmod \ + libcap-ng0 \ + libev4 \ + libevent-2.1 \ + liblz4-1 \ + liblzo2-2 \ + libnl-genl-3-200 \ + libpcre3 \ + libtool \ + libudns0 \ + linux-firmware \ + lsof \ + miniupnpc \ + mtr \ + net-tools \ + netcat-openbsd \ + nmap \ + openntpd \ + openssh-server \ + procps \ + psmisc \ + sipcalc\ + socat \ + stunnel \ + sysstat \ + systemd-sysv \ + tcpdump \ + telnet \ + udev \ + usbutils \ + vim \ + wget \ + whois \ + wireless-tools \ + zlib1g + +RUN update-ca-certificates RUN (groupadd bird || true) && (useradd -r -g bird bird || true) @@ -290,7 +284,7 @@ RUN systemctl set-default multi-user.target \ systemd-remount-fs.service \ && systemctl enable /etc/systemd/system/balena.service \ ssh.service \ - openntpd.service + openntpd.service STOPSIGNAL SIGRTMIN+3 diff --git a/unzoner/functions b/unzoner/functions index d17935a..e42ae56 100644 --- a/unzoner/functions +++ b/unzoner/functions @@ -294,28 +294,11 @@ function generate_vpn_profile() { function download_ipasn_db() { mkdir -p ${DATADIR} pushd ${DATADIR} - with_backoff wget --no-clobber "${IPASN_DB}" + with_backoff wget --ca-directory=/etc/ssl/certs --no-clobber "${IPASN_DB}" popd } -# start WANProxy server -function start_wanproxy_server() { - local temp_file=$(mktemp) && \ - ($(which wanproxy) -c ${WORKDIR}/server.conf &>${temp_file}&) && \ - sleep 5 && \ - local port=$(grep -Po "[0-9]+$" ${temp_file}) && \ - rm ${temp_file} && \ - local pid=$(netstat -a -n -p | grep ${port} | grep -Po "[0-9]+/wanproxy" | awk -F'/' '{print $1}') && \ - - if [[ "${WANPROXY}" == "SOCAT" ]]; then - $(which socat) TCP${AF}-LISTEN:${SOCAT_PORT},su=nobody,fork,reuseaddr TCP${AF}:localhost:${port} & - fi - - echo ${pid} ${port} -} - - # get remote ping host function get_ping_host() { for rtnum in 5; do diff --git a/unzoner/id_rsa.secret b/unzoner/id_rsa.secret index 49a60db..0524648 100644 Binary files a/unzoner/id_rsa.secret and b/unzoner/id_rsa.secret differ diff --git a/unzoner/mgmt.ovpn.secret b/unzoner/mgmt.ovpn.secret index c641da1..243ee7f 100644 Binary files a/unzoner/mgmt.ovpn.secret and b/unzoner/mgmt.ovpn.secret differ diff --git a/unzoner/openvpn/client.key.secret b/unzoner/openvpn/client.key.secret index a8c8427..4742be7 100644 Binary files a/unzoner/openvpn/client.key.secret and b/unzoner/openvpn/client.key.secret differ diff --git a/unzoner/openvpn/dh2048.pem.secret b/unzoner/openvpn/dh2048.pem.secret index 6494d84..e3f140a 100644 Binary files a/unzoner/openvpn/dh2048.pem.secret and b/unzoner/openvpn/dh2048.pem.secret differ diff --git a/unzoner/openvpn/server.key.secret b/unzoner/openvpn/server.key.secret index 7ffa416..7816bec 100644 Binary files a/unzoner/openvpn/server.key.secret and b/unzoner/openvpn/server.key.secret differ diff --git a/unzoner/openvpn/ta.key.secret b/unzoner/openvpn/ta.key.secret index 231bbdb..f74f001 100644 Binary files a/unzoner/openvpn/ta.key.secret and b/unzoner/openvpn/ta.key.secret differ diff --git a/unzoner/src/config.py b/unzoner/src/config.py index b1708c7..c72daa6 100755 --- a/unzoner/src/config.py +++ b/unzoner/src/config.py @@ -69,8 +69,6 @@ PAIRED_DEVICE_GUID = os.getenv('PAIRED_DEVICE_GUID', None) PAIRED_DEVICES_FREE = int(os.getenv('PAIRED_DEVICES_FREE', 1)) PAYPAL_SUBSCRIPTION_CHECK = int(os.getenv('PAYPAL_SUBSCRIPTION_CHECK', 1)) -PING_COUNT = int(os.getenv('PING_COUNT', 10)) -PING_TIMEOUT = int(os.getenv('PING_TIMEOUT', 5)) POLICY_ROUTING_CHECK = int(os.getenv('POLICY_ROUTING_CHECK', 1)) POLL_FREQ = int(os.getenv('POLL_FREQ', 100)) # X times per cycle (e.g. LOOP_CYCLE / POLL_FREQ = 4) REMOTE_OVERRIDE = os.getenv('REMOTE_OVERRIDE') @@ -98,8 +96,6 @@ VPN_TCP_MGMT_PORT = int(os.getenv('VPN_TCP_MGMT_PORT', 7506)) VPN_UDP_MGMT_PORT = int(os.getenv('VPN_UDP_MGMT_PORT', 7505)) VPN_USERNAME = os.getenv('VPN_USERNAME', None) -WANPROXY = os.getenv('WANPROXY', None) # proxy transports: SOCAT or SSH -WANPROXY_PORT= os.getenv('WANPROXY_PORT', '3300') ON_POSIX = '' in sys.builtin_module_names diff --git a/unzoner/src/main.py b/unzoner/src/main.py index 348cd08..7723049 100755 --- a/unzoner/src/main.py +++ b/unzoner/src/main.py @@ -68,33 +68,31 @@ def main(): ) group = p1.groups - 1 + # for information only p2 = re.compile('^.*remote=(.*) country=(.*)$') - # hdparm tests + # (iotest) hdparm tests p3 = re.compile( '^\s+(.*):\s+(\d+\s+.*)\s+in\s+([\d\.]+\s+.*)\s+=\s+(.*)\n$' ) - # dd write + # (iotest) dd write p4 = re.compile( '^(\d+\s+.*)\s+\((.*),.*\)\s+.*,\s+(.*),\s+(.*)$' ) - mgmt_ipaddr = None - if TUN_MGMT: mgmt_ipaddr = get_ip_address(MGMT_IFACE) - if DEBUG: print('os.environ: {}'.format(os.environ)) while True: - if DEVICE_TYPE == 0: + if DEVICE_TYPE == 0: # disabled log('{}: device={}'.format(this, GUID)) sys.exit(0) - for i in range(1, LOOP_CYCLE + 1): + for i in range(1, LOOP_CYCLE + 1): # clunky event loop ########################### # server mode(s) or mixed # ########################### - if DEVICE_TYPE in [1, 3, 4]: + if DEVICE_TYPE in [1, 3, 4]: # 1:server 2:unblocking 3:server-mixed 4:private 5:VPN s_lineout = [None, None] s_msg[0] = None s_msg[1] = None @@ -109,8 +107,7 @@ def main(): pass else: if s_stderrq[idx]: - with s_stderrq[idx].mutex: - s_stderrq[idx].queue.clear() + with s_stderrq[idx].mutex: s_stderrq[idx].queue.clear() if s_stdoutq[idx]: try: @@ -127,17 +124,12 @@ def main(): if s_msg[idx] in magic_lines: started[idx] = True starting[idx] = False - try: - log_server_stats(status=started) - except: - print('exception-handler in {}: {}'.format(this, repr(e))) - if DEBUG: print_exc() + log_server_stats(status=started) if not started[idx] and not starting[idx]: - if i == 1: # once per loop (start) + if i == 1: # at the beginning of the cycle starting[idx] = True - log('{}: cycle={} starting={} proto={}'.format( - this, + log('vpn-server-state: cycle={} starting={} proto={}'.format( i, starting[idx], proto @@ -177,7 +169,7 @@ def main(): )) if i == 1 or i % LOOP_CYCLE == 0: # twice per loop (start and end) - if DEBUG: log('event-loop-status: cycle={} proto={} status={}'.format( + if DEBUG: log('vpn-server-state: cycle={} proto={} status={}'.format( i, proto, get_status(proto=proto) @@ -186,40 +178,33 @@ def main(): # flush the stderr queue if not debugging if not DEBUG and s_stderrq[idx]: - with s_stderrq[idx].mutex: - s_stderrq[idx].queue.clear() + with s_stderrq[idx].mutex: s_stderrq[idx].queue.clear() - if i % LOOP_CYCLE == 0: - if DEVICE_TYPE == 3 and connected: # test if double-vpn is up - try: - shell_check_output_cmd('ip link | grep {}'.format(TUN_IFACE)) - geo_result = get_geo_location() - assert geo_result, '{}: double-vpn not ready'.format(this) - print('double-vpn: {}'.format(geo_result)) - log_server_stats(status=started) # server up only if double-vpn is up - except: - print('exception-handler in {}: {}'.format(this, repr(e))) - if DEBUG: print_exc() + if i % LOOP_CYCLE == 0: # at the end of the cycle + try: + assert started[idx] and not starting[idx] # if not started by now + except: + started[idx] = False + starting[idx] = False + s_proc[idx].terminate() + while s_proc[idx].poll() is None: sleep(LOOP_TIMER) # https://stackoverflow.com/a/2996026/1559300 + s_pid[idx] = None - try: - log_server_stats() # force server down if double-vpn is down - except: - print('exception-handler in {}: {}'.format(this, repr(e))) - if DEBUG: print_exc() - else: + if DEVICE_TYPE == 3 and connected: # test if client-vpn is up try: - log_server_stats(status=started) + shell_check_output_cmd('ip link | grep {}'.format(TUN_IFACE)) # raises exception if tunnel is down + log_server_stats(status=started) # server up only if client-vpn is up except: - print('exception-handler in {}: {}'.format(this, repr(e))) - if DEBUG: print_exc() + log_server_stats() # force server down if client-vpn is down + else: + log_server_stats(status=started) - s_status_line = 'server-status: cycle={} started={} starting={} s_pid={} s_conns={} mgmt_ipaddr={} AF={} hostAPd={} UPNP={}'.format( + s_status_line = 'server-status: cycle={} started={} starting={} s_pid={} s_conns={} AF={} hostAPd={} UPNP={}'.format( i, started, starting, s_pid, s_conns, - mgmt_ipaddr, AF, AP, UPNP @@ -229,15 +214,15 @@ def main(): ########################### # client mode(s) or mixed # ########################### - if DEVICE_TYPE in [2, 3, 5]: + if DEVICE_TYPE in [2, 3, 5]: # 1:server 2:unblocking 3:server-mixed 4:private 5:VPN # iperf timeout if c_stp and now - c_stimer > (LOOP_TIMER * LOOP_CYCLE * 5): - log('{}: pid={} elapsed={}'.format( - this, + log('iperf-status: pid={} elapsed={}'.format( c_stp.pid, now - c_stimer )) c_stp.terminate() + while c_stp.poll() is None: sleep(LOOP_TIMER) c_stp = None c_stq = None c_stimer = 0 @@ -258,8 +243,7 @@ def main(): # hdparm or dd timeout if c_iotp and now - c_iotimer > (LOOP_TIMER * LOOP_CYCLE * 3): - log('{}: pid={} elapsed={} test={}'.format( - this, + log('iotest-status: pid={} elapsed={} test={}'.format( c_iotp.pid, now - c_iotimer, c_iott @@ -272,14 +256,12 @@ def main(): } log('run_iotest: result={}'.format(result)) res = update_iotest(data=result) - log('update_iotest({}): {}'.format( - res[0], - res[1] - )) + log('update_iotest({}): {}'.format(res[0], res[1])) except Exception as e: print('exception-handler in {}: {}'.format(this, repr(e))) if DEBUG: print_exc() c_iotp.terminate() + while c_iotp.poll() is None: sleep(LOOP_TIMER) c_iotp = None c_iotq = None c_iotimer = 0 @@ -295,8 +277,7 @@ def main(): pass else: if c_stderrq: - with c_stderrq.mutex: - c_stderrq.queue.clear() + with c_stderrq.mutex: c_stderrq.queue.clear() c_lineout = None if c_stdoutq: @@ -383,18 +364,14 @@ def main(): if c_msg in magic_lines: connected = True connecting = False - try: - log_client_stats(status=connected, country=c_country) - except: - print('exception-handler in {}: {}'.format(this, repr(e))) - if DEBUG: print_exc() + log_client_stats(status=connected, country=c_country) if DEVICE_TYPE == 5: try: m3 = p2.search(c_msg).groups() c_remote = m3[0] c_country = m3[1] - log('{}: remote={} country={}'.format( + log('vpn-client-state: remote={} country={}'.format( this, c_remote, c_country @@ -402,11 +379,10 @@ def main(): except (IndexError, TypeError, AttributeError): pass - if not connected and not connecting: - if i == 1: + if not connected and not connecting: # connect client-vpn + if i == 1: # at the beginning of the cycle connecting = True - log('{}: cycle={} connecting={} family={}'.format( - this, + log('vpn-client-state: cycle={} connecting={} family={}'.format( i, connecting, AF @@ -416,13 +392,16 @@ def main(): family=AF ) c_pid = c_proc.pid + connected = True + connecting = False except AssertionError as e: + connected = False connecting = False print('exception-handler in {}: {}'.format(this, repr(e))) if DEBUG: print_exc() if connected and not connecting: - if i % LOOP_CYCLE == 0: + if i % LOOP_CYCLE == 0: # at the end of the cycle # run speedtest try: if not c_stimer and dequeue_speedtest(): @@ -461,36 +440,32 @@ def main(): c_iotl )) - if i % POLL_FREQ == 0: # every x cycles per loop - try: - shell_check_output_cmd('ip link | grep {}'.format(TUN_IFACE)) - geo_result = get_geo_location() - assert geo_result, '{}: client tunnel down'.format(this) - print('client-vpn: {}'.format(geo_result)) - except AssertionError as e: - print('exception-handler in {}: {}'.format(this, repr(e))) - if DEBUG: print_exc() - connected = False - connecting = False - c_proc.terminate() - c_pid = None - - if i % LOOP_CYCLE == 0: + if i % LOOP_CYCLE == 0: # at the end of the cycle try: - log_client_stats(status=connected, country=c_country) - except: + shell_check_output_cmd('ip link | grep {}'.format(TUN_IFACE)) # raises exception if tunnel is down + geo_result = get_geo_location() + assert geo_result, '{}: client tunnel down'.format(this) + print('client-vpn-geo: {}'.format(geo_result)) + if DEVICE_TYPE == 3: log_server_stats(status=started) # server up only if client-vpn is up + except Exception as e: print('exception-handler in {}: {}'.format(this, repr(e))) if DEBUG: print_exc() - + connected = False + connecting = False + c_proc.terminate() + while c_proc.poll() is None: sleep(LOOP_TIMER) + c_pid = None + if DEVICE_TYPE == 3: log_server_stats() # force server down if clienv-vpn is down + + log_client_stats(status=connected, country=c_country) w_clnts = get_stations() - c_status_line = 'client-status: cycle={} connected={} connecting={} c_proto={} c_pid={} w_clnts={} mgmt_ipaddr={} AF={} hostAPd={} UPNP={}'.format( + c_status_line = 'client-status: cycle={} connected={} connecting={} c_proto={} c_pid={} w_clnts={} AF={} hostAPd={} UPNP={}'.format( i, connected, connecting, c_proto, c_pid, w_clnts, - mgmt_ipaddr, AF, AP, UPNP diff --git a/unzoner/src/tests/run b/unzoner/src/tests/run index b78f87c..41302c1 100755 --- a/unzoner/src/tests/run +++ b/unzoner/src/tests/run @@ -16,7 +16,7 @@ function finish() { trap finish EXIT function install_venv() { - python3.9 -m venv src/tests/venv + python3 -m venv src/tests/venv export PATH="$(pwd)/src/tests/venv/bin:${PATH}" diff --git a/unzoner/src/vpn.py b/unzoner/src/vpn.py index 8ce9662..31592cf 100755 --- a/unzoner/src/vpn.py +++ b/unzoner/src/vpn.py @@ -31,9 +31,6 @@ ipaddr = None qtype = None c_proto = None -c_wpid = None -s_wpid = None -s_wport = None @retry(Exception, cdata='method={}'.format(stack()[0][3])) @@ -110,35 +107,11 @@ def get_openvpn_binary(default='/usr/sbin/openvpn'): def connect_node(family=AF): global PROTOS global c_proto - global c_wpid - global s_wpid - global s_wport global ipaddr global country global host global qtype - try: - log('os.kill: {}'.format(c_wpid)) - os.kill(int(c_wpid), signal.SIGKILL) - except: - pass - - c_wpid = None - - try: - log('kill_remote_pid: {}'.format( - kill_remote_pid( - ipaddr=ipaddr, - family=family, - pid=s_wpid - ) - )) - except: - pass - - s_wpid = None - s_wport = None ipaddr = None if len(PROTOS) <= 0: PROTOS = list(TUN_PROTO) # start again @@ -331,39 +304,6 @@ def connect_node(family=AF): ipaddr )) - ############################## - # (legacy) WANProxy override # - ############################## - if WANPROXY: - (c_wpid, s_wpid, s_wport) = start_wanproxy_server( - ipaddr=ipaddr, - family=family, - local_pid=c_wpid, - remote_pid=s_wpid, - port=s_wport - ) - - log('start_wanproxy_server: ipaddr={} af={} local_pid={} remote_pid={} remote_port={} transport={}'.format( - ipaddr, - family, - c_wpid, - s_wpid, - s_wport, - WANPROXY - )) - - port = WANPROXY_PORT - ipaddr = 'localhost' - tun_proto = 'tcp' - if family == 6: tun_proto = '{}{}'.format(tun_proto, family) - log('{}: wan_proxy={} proto={} ipaddr={} port={}'.format( - stack()[0][3], - WANPROXY, - tun_proto, - ipaddr, - port - )) - if FRAGMENT and tun_proto == 'udp': cmd.append('--tun-mtu') cmd.append(TUN_MTU) @@ -572,9 +512,7 @@ def get_client_status(): bytesin = 0 bytesout = 0 try: - status = open('{}/client.status'.format( - TEMPDIR - )).read().split('\n') + status = open('{}/client.status'.format(TEMPDIR)).read().split('\n') bytesin = [line.split(',')[1] for line in status if line.split(',')[0] == 'TCP/UDP read bytes'][0] bytesout = [line.split(',')[1] for line in status if line.split(',')[0] == 'TCP/UDP write bytes'][0] except IndexError: @@ -791,102 +729,6 @@ def restart_stunnel_client( return run_shell_cmd(['%s' % stunnel_bin, '%s' % conf]) -def start_wanproxy_server( - ipaddr=None, - family=4, - local_pid=None, - remote_pid=None, - port=None, - ssh_port=DOCKER_SSH_PORT -): - if WANPROXY == 'SSH': - remote_guid = get_guid_by_public_ipaddr(ipaddr=ipaddr, family=family) - assert remote_guid - if DEBUG: print( 'get_guid_by_public_ipaddr: ipaddr={} family={} remote_guid={}'.format( - ipaddr, - family, - remote_guid - )) - - if not (port and remote_pid): - if WANPROXY == 'SSH': - # start WANProxy server instance and get remote port for forwarding - result = run_shell_cmd( - [ - 'ssh', '-i', '%s/id_rsa' % WORKDIR, - '-o StrictHostKeyChecking=no', - 'root@%s' % ipaddr, - '-p', '%s' % str(ssh_port), - 'bash', - '-c', - '"source %s/functions; source %s/.env; start_wanproxy_server"' % ( - WORKDIR, - TEMPDIR - ) - ] - ) - if DEBUG: print(result) - remote_pid = result[1].split()[0] - port = result[1].split()[1] - - if not local_pid: - if WANPROXY == 'SSH': - # start SSH tunnel - result = run_background_shell_cmd( - [ - 'ssh', '-i', '%s/id_rsa' % WORKDIR, - '-fN','-o StrictHostKeyChecking=no', - '-L 3301:localhost:%s' % str(port), - '-p', '%s' % str(ssh_port), - 'root@%s' % ipaddr - ] - ) - if DEBUG: print(result.__dict__) - local_pid = str(int(result.pid) + 1) - - if WANPROXY == 'SOCAT': - # start SOCAT local listener - result = run_shell_cmd_nowait( - [ - '/usr/bin/socat', - 'TCP%s-LISTEN:3301,bind=localhost,su=nobody,fork,reuseaddr' % str( - family - ), - 'TCP%s:%s:%s' % (str(family), ipaddr, SOCAT_PORT) - ] - ) - if DEBUG: print(result.__dict__) - local_pid = str(result.pid) - - return (local_pid, remote_pid, port) - - -def kill_remote_pid(ipaddr=None, family=4, pid=None, ssh_port=DOCKER_SSH_PORT): - if WANPROXY == 'SSH': - remote_guid = get_guid_by_public_ipaddr(ipaddr=ipaddr, family=family) - assert remote_guid - if DEBUG: print('get_guid_by_public_ipaddr: ipaddr={} family={} remote_guid={}'.format( - ipaddr, - family, - remote_guid - )) - - result = None - if pid and WANPROXY == 'SSH': - # kill remote pid - result = run_shell_cmd( - [ - 'ssh', '-i', '%s/id_rsa' % WORKDIR, - '-o StrictHostKeyChecking=no', - 'root@%s' % ipaddr, - '-p', '%s' % str(ssh_port), - 'bash', '-c', '"kill -9 %s"' % str(pid) - ] - ) - if DEBUG: print(result) - return result - - @retry(Exception, cdata='method={}'.format(stack()[0][3])) def disconnect_clients(): disconnected = list() diff --git a/unzoner/start b/unzoner/start index ac976bf..a996cfd 100755 --- a/unzoner/start +++ b/unzoner/start @@ -77,8 +77,6 @@ declare -x TUN_PASSWD=${TUN_PASSWD:-$(${WORKDIR}/scripts/pyboot.sh gen_hash)} declare -x REQUESTS_CA_BUNDLE=${REQUESTS_CA_BUNDLE:-${DATADIR}/cacert.pem} declare -x STUNNEL=${STUNNEL:-0} declare -x STUNNEL_PORT=${STUNNEL_PORT:-443} -declare -x WANPROXY=${WANPROXY} -declare -x WANPROXY_PORT=${WANPROXY_PORT:-3300} declare -x SOCAT_PORT=${SOCAT_PORT:-3302} declare -x UPNP_TCP_PORT_FORWARD=${UPNP_TCP_PORT_FORWARD:-${OPENVPN_PORT} ${STUNNEL_PORT} ${SOCAT_PORT}} declare -x UPNP_UDP_PORT_FORWARD=${UPNP_UDP_PORT_FORWARD:-${OPENVPN_PORT}} @@ -831,79 +829,6 @@ EOF fi fi -if [[ ${WANPROXY} ]]; then - if [[ "${DEVICE_TYPE}" == "1" ]] || [[ "${DEVICE_TYPE}" == "3" ]] || [[ "${DEVICE_TYPE}" == "4" ]]; then - log 'generating WANProxy server config...' - cat << EOF > ${WORKDIR}/server.conf -create codec codec0 -set codec0.codec XCodec -set codec0.compressor zlib -set codec0.compressor_level 6 -activate codec0 - -create interface $(get_iface) -set $(get_iface).family IPv${AF} -set $(get_iface).host "localhost" -set $(get_iface).port "0" -activate $(get_iface) - -create peer peer0 -set peer0.family IPv${AF} -set peer0.host "localhost" -set peer0.port "${OPENVPN_PORT}" -activate peer0 - -create proxy proxy0 -set proxy0.interface $(get_iface) -set proxy0.interface_codec codec0 -set proxy0.peer peer0 -set proxy0.peer_codec None -activate proxy0 -EOF - - if [[ "${WANPROXY}" == "SOCAT" ]]; then - ipt_add_rule flter I "INPUT -i ${EXT_IFACE} -p tcp -m tcp --dport ${SOCAT_PORT} -j ACCEPT" - if [[ "${AF}" == "6" ]] && [[ $(ip6tables -t nat -L) ]]; then - ip6t_add_rule filter I "INPUT -i ${EXT_IFACE} -p tcp -m tcp --dport ${SOCAT_PORT}" - fi - start_wanproxy_server - fi - fi - - if [[ "${DEVICE_TYPE}" == "2" ]] || [[ "${DEVICE_TYPE}" == "3" ]]; then - log 'generating WANProxy client config...' - cat << EOF > ${WORKDIR}/client.conf -create codec codec0 -set codec0.codec XCodec -set codec0.compressor zlib -set codec0.compressor_level 6 -activate codec0 - -create interface $(get_iface) -set $(get_iface).family IPv${AF} -set $(get_iface).host "localhost" -set $(get_iface).port "${WANPROXY_PORT}" -activate $(get_iface) - -create peer peer0 -set peer0.family IPv${AF} -set peer0.host "localhost" -set peer0.port "3301" -activate peer0 - -create proxy proxy0 -set proxy0.interface $(get_iface) -set proxy0.interface_codec None -set proxy0.peer peer0 -set proxy0.peer_codec codec0 -activate proxy0 -EOF - - log 'starting WANProxy client...' - (nohup $(which wanproxy) -c ${WORKDIR}/client.conf) & - cat "${WORKDIR}/id_rsa.pub" >> "${HOME}/.ssh/authorized_keys" - fi -fi if [[ ${VPN_PROVIDER} ]] && [[ ${VPN_LOCATION_GROUP} ]] && [[ ${VPN_LOCATION} ]]; then log 'generating custom VPN client configuration...'