Social media event detector

detector/Dockerfile

@@ -0,0 +1,21 @@
+
+FROM debian:bullseye
+ENV PYTHONUNBUFFERED 1
+# ENV PYTHONPATH /opt/detector
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN mkdir /scripts
+COPY build_runner.sh /scripts
+# COPY debian/ooni_download_geoip.py /scripts
+# COPY api.conf.example /scripts
+WORKDIR /scripts
+
+# Run runner setup
+RUN ./build_runner.sh
+
+# # Download geoip files
+# RUN ./ooni_download_geoip.py
+
+RUN rm -rf /scripts
+
+WORKDIR /opt/detector

detector/build_runner.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# WARNING: run only in a dedicated container
+# Prepares a container to run the detector
+# Called from spawnrunner or docker
+#
+set -eu
+export DEBIAN_FRONTEND=noninteractive
+
+#echo 'deb http://deb.debian.org/debian bullseye-backports main' \
+#  > /etc/apt/sources.list.d/backports.list
+
+# Install ca-certificates and gnupg first
+apt-get update
+apt-get install --no-install-recommends -y ca-certificates gnupg locales apt-transport-https dirmngr
+locale-gen en_US.UTF-8
+
+# Set up OONI archive
+#echo 'deb http://deb-ci.ooni.org unstable main' \
+#  > /etc/apt/sources.list.d/ooni.list
+#apt-key adv --keyserver hkp://keyserver.ubuntu.com \
+#  --recv-keys "B5A08F01796E7F521861B449372D1FF271F2DD50"
+
+apt-get update -q
+#apt-get dist-upgrade
+# Keep this in sync with debian/control
+# hint: grep debdeps **/*.py
+apt-get install --no-install-recommends -qy \
+  python3-clickhouse-driver \
+  python3-feedgenerator \
+  python3-geoip2 \
+  python3-mock \
+  python3-numpy \
+  python3-pandas \
+  python3-pytest \
+  python3-pytest-cov \
+  python3-pytest-mock \
+  python3-setuptools \
+  python3-statsd \
+  python3-systemd \
+  python3-ujson
+apt-get autoremove -y
+apt-get clean
+rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+mkdir -p /etc/ooni/

detector/debian/changelog

@@ -1,3 +1,9 @@
+detector (0.3) unstable; urgency=medium
+
+  * Implement event detector for social media using Clickhouse
+
+ -- Federico Ceratto <[email protected]>  Fri, 11 Nov 2022 18:08:26 +0000
+
 detector (0.2) unstable; urgency=medium
 
   * run webapp as a dedicated service

detector/debian/control

@@ -2,33 +2,23 @@ Source: detector
 Section: python
 Priority: optional
 Maintainer: Federico Ceratto <[email protected]>
-Build-Depends: debhelper-compat (= 12),
+Build-Depends: debhelper-compat (= 13),
  python3,
- dh-systemd (>= 1.5),
  dh-python,
- python3-boto3,
- python3-lz4,
- python3-paramiko,
- python3-psycopg2,
+ python3-clickhouse-driver,
  python3-setuptools,
  python3-statsd,
- python3-systemd,
- python3-ujson
-Standards-Version: 4.1.3
+ python3-systemd
+Standards-Version: 4.6.0
 
 Package: detector
 Architecture: all
 Depends: ${misc:Depends},
  ${python3:Depends},
- python3-boto3,
- python3-lz4,
- python3-paramiko,
- python3-psycopg2,
+ python3-clickhouse-driver,
  python3-setuptools,
  python3-statsd,
- python3-systemd,
- python3-ujson,
- nginx
+ python3-systemd
 Suggests:
  bpython3,
  python3-pytest,

detector/debian/detector.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run event detector
+Requires=detector.service
+
+[Timer]
+Unit=detector.service
+OnCalendar=*-*-* *:10:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target

detector/debian/postinst

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -e
+
+case "$1" in
+    configure)
+        #addgroup --system --quiet fastpath
+        #adduser --system --quiet --ingroup fastpath --no-create-home --home /var/lib/fastpath fastpath
+        mkdir -p /var/lib/detector
+        chown fastpath /var/lib/detector -R
+    ;;
+
+    abort-upgrade|abort-remove|abort-deconfigure)
+    ;;
+
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+    ;;
+esac
+
+# dh_installdeb will replace this with shell code automatically
+# generated by other debhelper scripts.
+
+#DEBHELPER#
+
+exit 0