diff --git a/detector/debian/changelog b/detector/debian/changelog new file mode 100644 index 00000000..fa2ee243 --- /dev/null +++ b/detector/debian/changelog @@ -0,0 +1,11 @@ +detector (0.2) unstable; urgency=medium + + * run webapp as a dedicated service + + -- Federico Ceratto Tue, 26 Nov 2019 18:58:45 +0000 + +detector (0.1) unstable; urgency=medium + + * event detector + + -- Federico Ceratto Tue, 26 Nov 2019 18:58:23 +0000 diff --git a/detector/debian/control b/detector/debian/control new file mode 100644 index 00000000..8c1fb036 --- /dev/null +++ b/detector/debian/control @@ -0,0 +1,37 @@ +Source: detector +Section: python +Priority: optional +Maintainer: Federico Ceratto +Build-Depends: debhelper-compat (= 12), + python3, + dh-systemd (>= 1.5), + dh-python, + python3-boto3, + python3-lz4, + python3-paramiko, + python3-psycopg2, + python3-setuptools, + python3-statsd, + python3-systemd, + python3-ujson +Standards-Version: 4.1.3 + +Package: detector +Architecture: all +Depends: ${misc:Depends}, + ${python3:Depends}, + python3-boto3, + python3-lz4, + python3-paramiko, + python3-psycopg2, + python3-setuptools, + python3-statsd, + python3-systemd, + python3-ujson, + nginx +Suggests: + bpython3, + python3-pytest, + python3-pytest-benchmark +Description: OONI Event Detector + OONI Event Detector diff --git a/detector/debian/detector-webapp.service b/detector/debian/detector-webapp.service new file mode 100644 index 00000000..2c9be94e --- /dev/null +++ b/detector/debian/detector-webapp.service @@ -0,0 +1,36 @@ +[Unit] +Description=OONI Detector Webapp +Wants=network-online.target +After=network-online.target + +[Service] +ExecStart=/usr/bin/detector --webapp +Restart=on-abort +Type=simple +RestartSec=2s +WorkingDirectory=/var/lib/detector + +User=fastpath +Group=fastpath +ReadOnlyDirectories=/ +ReadWriteDirectories=/proc/self + +StandardOutput=syslog+console +StandardError=syslog+console + +PermissionsStartOnly=true +LimitNOFILE=65536 + +# Hardening +CapabilityBoundingSet=CAP_SETUID CAP_SETGID +SystemCallFilter=~@clock @debug @cpu-emulation @keyring @module @mount @obsolete @raw-io @reboot @swap +NoNewPrivileges=yes +PrivateDevices=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=full +ProtectKernelModules=yes +ProtectKernelTunables=yes + +[Install] +WantedBy=multi-user.target diff --git a/detector/debian/detector.service b/detector/debian/detector.service new file mode 100644 index 00000000..c1bb5ac7 --- /dev/null +++ b/detector/debian/detector.service @@ -0,0 +1,37 @@ +[Unit] +Description=OONI Event Detector +Wants=network-online.target +After=network-online.target + +[Service] +ExecStart=/usr/bin/detector +Restart=on-abort +Type=simple +RestartSec=2s +WorkingDirectory=/var/lib/detector + +User=fastpath +Group=fastpath +ReadOnlyDirectories=/ +ReadWriteDirectories=/proc/self +ReadWriteDirectories=/var/lib/detector + +StandardOutput=syslog+console +StandardError=syslog+console + +PermissionsStartOnly=true +LimitNOFILE=65536 + +# Hardening +CapabilityBoundingSet=CAP_SETUID CAP_SETGID +SystemCallFilter=~@clock @debug @cpu-emulation @keyring @module @mount @obsolete @raw-io @reboot @swap +NoNewPrivileges=yes +PrivateDevices=yes +PrivateTmp=yes +ProtectHome=yes +ProtectSystem=full +ProtectKernelModules=yes +ProtectKernelTunables=yes + +[Install] +WantedBy=multi-user.target diff --git a/detector/debian/rules b/detector/debian/rules new file mode 100755 index 00000000..8b979857 --- /dev/null +++ b/detector/debian/rules @@ -0,0 +1,9 @@ +#!/usr/bin/make -f +export DH_VERBOSE = 1 + +%: + dh $@ --buildsystem=pybuild + +override_dh_installsystemd: + dh_installsystemd --no-restart-on-upgrade --name detector + dh_installsystemd --no-restart-on-upgrade --name detector-webapp diff --git a/detector/debian/source/format b/detector/debian/source/format new file mode 100644 index 00000000..163aaf8d --- /dev/null +++ b/detector/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/detector/detector/__init__.py b/detector/detector/__init__.py new file mode 100644 index 00000000..792d6005 --- /dev/null +++ b/detector/detector/__init__.py @@ -0,0 +1 @@ +# diff --git a/detector/detector/data/README.adoc b/detector/detector/data/README.adoc new file mode 100644 index 00000000..27ba63cf --- /dev/null +++ b/detector/detector/data/README.adoc @@ -0,0 +1 @@ +The country-list.json file is generated by https://github.com/ooni/country-util diff --git a/detector/detector/data/country-list.json b/detector/detector/data/country-list.json new file mode 100644 index 00000000..cb30c255 --- /dev/null +++ b/detector/detector/data/country-list.json @@ -0,0 +1 @@ +[{"iso3166_alpha2": "TW", "iso3166_alpha3": "TWN", "iso3166_num": "158", "iso3166_name": "Taiwan", "name": "Taiwan", "languages": ["zh-TW", "zh", "nan", "hak"], "tld": ".tw", "capital": "Taipei", "region_code": "142", "sub_region_code": "156"}, {"iso3166_alpha2": "AF", "iso3166_alpha3": "AFG", "iso3166_num": "004", "iso3166_name": "Afghanistan", "name": "Afghanistan", "languages": ["fa-AF", "ps", "uz-AF", "tk"], "tld": ".af", "capital": "Kabul", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "AL", "iso3166_alpha3": "ALB", "iso3166_num": "008", "iso3166_name": "Albania", "name": "Albania", "languages": ["sq", "el"], "tld": ".al", "capital": "Tirana", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "DZ", "iso3166_alpha3": "DZA", "iso3166_num": "012", "iso3166_name": "Algeria", "name": "Algeria", "languages": ["ar-DZ"], "tld": ".dz", "capital": "Algiers", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "AS", "iso3166_alpha3": "ASM", "iso3166_num": "016", "iso3166_name": "American Samoa", "name": "American Samoa", "languages": ["en-AS", "sm", "to"], "tld": ".as", "capital": "Pago Pago", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "AD", "iso3166_alpha3": "AND", "iso3166_num": "020", "iso3166_name": "Andorra", "name": "Andorra", "languages": ["ca"], "tld": ".ad", "capital": "Andorra la Vella", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "AO", "iso3166_alpha3": "AGO", "iso3166_num": "024", "iso3166_name": "Angola", "name": "Angola", "languages": ["pt-AO"], "tld": ".ao", "capital": "Luanda", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "AI", "iso3166_alpha3": "AIA", "iso3166_num": "660", "iso3166_name": "Anguilla", "name": "Anguilla", "languages": ["en-AI"], "tld": ".ai", "capital": "The Valley", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AQ", "iso3166_alpha3": "ATA", "iso3166_num": "010", "iso3166_name": "Antarctica", "name": "Antarctica", "languages": [""], "tld": ".aq", "capital": "", "region_code": "", "sub_region_code": ""}, {"iso3166_alpha2": "AG", "iso3166_alpha3": "ATG", "iso3166_num": "028", "iso3166_name": "Antigua & Barbuda", "name": "Antigua & Barbuda", "languages": ["en-AG"], "tld": ".ag", "capital": "St. John's", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AR", "iso3166_alpha3": "ARG", "iso3166_num": "032", "iso3166_name": "Argentina", "name": "Argentina", "languages": ["es-AR", "en", "it", "de", "fr", "gn"], "tld": ".ar", "capital": "Buenos Aires", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AM", "iso3166_alpha3": "ARM", "iso3166_num": "051", "iso3166_name": "Armenia", "name": "Armenia", "languages": ["hy"], "tld": ".am", "capital": "Yerevan", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "AW", "iso3166_alpha3": "ABW", "iso3166_num": "533", "iso3166_name": "Aruba", "name": "Aruba", "languages": ["nl-AW", "pap", "es", "en"], "tld": ".aw", "capital": "Oranjestad", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "AU", "iso3166_alpha3": "AUS", "iso3166_num": "036", "iso3166_name": "Australia", "name": "Australia", "languages": ["en-AU"], "tld": ".au", "capital": "Canberra", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "AT", "iso3166_alpha3": "AUT", "iso3166_num": "040", "iso3166_name": "Austria", "name": "Austria", "languages": ["de-AT", "hr", "hu", "sl"], "tld": ".at", "capital": "Vienna", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "AZ", "iso3166_alpha3": "AZE", "iso3166_num": "031", "iso3166_name": "Azerbaijan", "name": "Azerbaijan", "languages": ["az", "ru", "hy"], "tld": ".az", "capital": "Baku", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "BS", "iso3166_alpha3": "BHS", "iso3166_num": "044", "iso3166_name": "Bahamas", "name": "Bahamas", "languages": ["en-BS"], "tld": ".bs", "capital": "Nassau", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BH", "iso3166_alpha3": "BHR", "iso3166_num": "048", "iso3166_name": "Bahrain", "name": "Bahrain", "languages": ["ar-BH", "en", "fa", "ur"], "tld": ".bh", "capital": "Manama", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "BD", "iso3166_alpha3": "BGD", "iso3166_num": "050", "iso3166_name": "Bangladesh", "name": "Bangladesh", "languages": ["bn-BD", "en"], "tld": ".bd", "capital": "Dhaka", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "BB", "iso3166_alpha3": "BRB", "iso3166_num": "052", "iso3166_name": "Barbados", "name": "Barbados", "languages": ["en-BB"], "tld": ".bb", "capital": "Bridgetown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BY", "iso3166_alpha3": "BLR", "iso3166_num": "112", "iso3166_name": "Belarus", "name": "Belarus", "languages": ["be", "ru"], "tld": ".by", "capital": "Minsk", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "BE", "iso3166_alpha3": "BEL", "iso3166_num": "056", "iso3166_name": "Belgium", "name": "Belgium", "languages": ["nl-BE", "fr-BE", "de-BE"], "tld": ".be", "capital": "Brussels", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "BZ", "iso3166_alpha3": "BLZ", "iso3166_num": "084", "iso3166_name": "Belize", "name": "Belize", "languages": ["en-BZ", "es"], "tld": ".bz", "capital": "Belmopan", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BJ", "iso3166_alpha3": "BEN", "iso3166_num": "204", "iso3166_name": "Benin", "name": "Benin", "languages": ["fr-BJ"], "tld": ".bj", "capital": "Porto-Novo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BM", "iso3166_alpha3": "BMU", "iso3166_num": "060", "iso3166_name": "Bermuda", "name": "Bermuda", "languages": ["en-BM", "pt"], "tld": ".bm", "capital": "Hamilton", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "BT", "iso3166_alpha3": "BTN", "iso3166_num": "064", "iso3166_name": "Bhutan", "name": "Bhutan", "languages": ["dz"], "tld": ".bt", "capital": "Thimphu", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "BO", "iso3166_alpha3": "BOL", "iso3166_num": "068", "iso3166_name": "Bolivia", "name": "Bolivia", "languages": ["es-BO", "qu", "ay"], "tld": ".bo", "capital": "Sucre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BQ", "iso3166_alpha3": "BES", "iso3166_num": "535", "iso3166_name": "Caribbean Netherlands", "name": "Caribbean Netherlands", "languages": ["nl", "pap", "en"], "tld": ".bq", "capital": "", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BA", "iso3166_alpha3": "BIH", "iso3166_num": "070", "iso3166_name": "Bosnia", "name": "Bosnia", "languages": ["bs", "hr-BA", "sr-BA"], "tld": ".ba", "capital": "Sarajevo", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "BW", "iso3166_alpha3": "BWA", "iso3166_num": "072", "iso3166_name": "Botswana", "name": "Botswana", "languages": ["en-BW", "tn-BW"], "tld": ".bw", "capital": "Gaborone", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BV", "iso3166_alpha3": "BVT", "iso3166_num": "074", "iso3166_name": "Bouvet Island", "name": "Bouvet Island", "languages": [""], "tld": ".bv", "capital": "", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BR", "iso3166_alpha3": "BRA", "iso3166_num": "076", "iso3166_name": "Brazil", "name": "Brazil", "languages": ["pt-BR", "es", "en", "fr"], "tld": ".br", "capital": "Brasilia", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "IO", "iso3166_alpha3": "IOT", "iso3166_num": "086", "iso3166_name": "British Indian Ocean Territory", "name": "British Indian Ocean Territory", "languages": ["en-IO"], "tld": ".io", "capital": "Diego Garcia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "VG", "iso3166_alpha3": "VGB", "iso3166_num": "092", "iso3166_name": "British Virgin Islands", "name": "British Virgin Islands", "languages": ["en-VG"], "tld": ".vg", "capital": "Road Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "BN", "iso3166_alpha3": "BRN", "iso3166_num": "096", "iso3166_name": "Brunei", "name": "Brunei", "languages": ["ms-BN", "en-BN"], "tld": ".bn", "capital": "Bandar Seri Begawan", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "BG", "iso3166_alpha3": "BGR", "iso3166_num": "100", "iso3166_name": "Bulgaria", "name": "Bulgaria", "languages": ["bg", "tr-BG", "rom"], "tld": ".bg", "capital": "Sofia", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "BF", "iso3166_alpha3": "BFA", "iso3166_num": "854", "iso3166_name": "Burkina Faso", "name": "Burkina Faso", "languages": ["fr-BF", "mos"], "tld": ".bf", "capital": "Ouagadougou", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BI", "iso3166_alpha3": "BDI", "iso3166_num": "108", "iso3166_name": "Burundi", "name": "Burundi", "languages": ["fr-BI", "rn"], "tld": ".bi", "capital": "Bujumbura", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CV", "iso3166_alpha3": "CPV", "iso3166_num": "132", "iso3166_name": "Cape Verde", "name": "Cape Verde", "languages": ["pt-CV"], "tld": ".cv", "capital": "Praia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KH", "iso3166_alpha3": "KHM", "iso3166_num": "116", "iso3166_name": "Cambodia", "name": "Cambodia", "languages": ["km", "fr", "en"], "tld": ".kh", "capital": "Phnom Penh", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "CM", "iso3166_alpha3": "CMR", "iso3166_num": "120", "iso3166_name": "Cameroon", "name": "Cameroon", "languages": ["en-CM", "fr-CM"], "tld": ".cm", "capital": "Yaounde", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CA", "iso3166_alpha3": "CAN", "iso3166_num": "124", "iso3166_name": "Canada", "name": "Canada", "languages": ["en-CA", "fr-CA", "iu"], "tld": ".ca", "capital": "Ottawa", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "KY", "iso3166_alpha3": "CYM", "iso3166_num": "136", "iso3166_name": "Cayman Islands", "name": "Cayman Islands", "languages": ["en-KY"], "tld": ".ky", "capital": "George Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CF", "iso3166_alpha3": "CAF", "iso3166_num": "140", "iso3166_name": "Central African Republic", "name": "Central African Republic", "languages": ["fr-CF", "sg", "ln", "kg"], "tld": ".cf", "capital": "Bangui", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "TD", "iso3166_alpha3": "TCD", "iso3166_num": "148", "iso3166_name": "Chad", "name": "Chad", "languages": ["fr-TD", "ar-TD", "sre"], "tld": ".td", "capital": "N'Djamena", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CL", "iso3166_alpha3": "CHL", "iso3166_num": "152", "iso3166_name": "Chile", "name": "Chile", "languages": ["es-CL"], "tld": ".cl", "capital": "Santiago", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CN", "iso3166_alpha3": "CHN", "iso3166_num": "156", "iso3166_name": "China", "name": "China", "languages": ["zh-CN", "yue", "wuu", "dta", "ug", "za"], "tld": ".cn", "capital": "Beijing", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "HK", "iso3166_alpha3": "HKG", "iso3166_num": "344", "iso3166_name": "Hong Kong", "name": "Hong Kong", "languages": ["zh-HK", "yue", "zh", "en"], "tld": ".hk", "capital": "Hong Kong", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "MO", "iso3166_alpha3": "MAC", "iso3166_num": "446", "iso3166_name": "Macau", "name": "Macao", "languages": ["zh", "zh-MO", "pt"], "tld": ".mo", "capital": "Macao", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "CX", "iso3166_alpha3": "CXR", "iso3166_num": "162", "iso3166_name": "Christmas Island", "name": "Christmas Island", "languages": ["en", "zh", "ms-CC"], "tld": ".cx", "capital": "Flying Fish Cove", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "CC", "iso3166_alpha3": "CCK", "iso3166_num": "166", "iso3166_name": "Cocos (Keeling) Islands", "name": "Cocos (Keeling) Islands", "languages": ["ms-CC", "en"], "tld": ".cc", "capital": "West Island", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "CO", "iso3166_alpha3": "COL", "iso3166_num": "170", "iso3166_name": "Colombia", "name": "Colombia", "languages": ["es-CO"], "tld": ".co", "capital": "Bogota", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "KM", "iso3166_alpha3": "COM", "iso3166_num": "174", "iso3166_name": "Comoros", "name": "Comoros", "languages": ["ar", "fr-KM"], "tld": ".km", "capital": "Moroni", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CG", "iso3166_alpha3": "COG", "iso3166_num": "178", "iso3166_name": "Congo - Brazzaville", "name": "Congo - Brazzaville", "languages": ["fr-CG", "kg", "ln-CG"], "tld": ".cg", "capital": "Brazzaville", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "CK", "iso3166_alpha3": "COK", "iso3166_num": "184", "iso3166_name": "Cook Islands", "name": "Cook Islands", "languages": ["en-CK", "mi"], "tld": ".ck", "capital": "Avarua", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "CR", "iso3166_alpha3": "CRI", "iso3166_num": "188", "iso3166_name": "Costa Rica", "name": "Costa Rica", "languages": ["es-CR", "en"], "tld": ".cr", "capital": "San Jose", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HR", "iso3166_alpha3": "HRV", "iso3166_num": "191", "iso3166_name": "Croatia", "name": "Croatia", "languages": ["hr-HR", "sr"], "tld": ".hr", "capital": "Zagreb", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "CU", "iso3166_alpha3": "CUB", "iso3166_num": "192", "iso3166_name": "Cuba", "name": "Cuba", "languages": ["es-CU", "pap"], "tld": ".cu", "capital": "Havana", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CW", "iso3166_alpha3": "CUW", "iso3166_num": "531", "iso3166_name": "Cura\u00e7ao", "name": "Cura\u00e7ao", "languages": ["nl", "pap"], "tld": ".cw", "capital": " Willemstad", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "CY", "iso3166_alpha3": "CYP", "iso3166_num": "196", "iso3166_name": "Cyprus", "name": "Cyprus", "languages": ["el-CY", "tr-CY", "en"], "tld": ".cy", "capital": "Nicosia", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "CZ", "iso3166_alpha3": "CZE", "iso3166_num": "203", "iso3166_name": "Czechia", "name": "Czechia", "languages": ["cs", "sk"], "tld": ".cz", "capital": "Prague", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "CI", "iso3166_alpha3": "CIV", "iso3166_num": "384", "iso3166_name": "C\u00f4te d\u2019Ivoire", "name": "C\u00f4te d\u2019Ivoire", "languages": ["fr-CI"], "tld": ".ci", "capital": "Yamoussoukro", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KP", "iso3166_alpha3": "PRK", "iso3166_num": "408", "iso3166_name": "North Korea", "name": "North Korea", "languages": ["ko-KP"], "tld": ".kp", "capital": "Pyongyang", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "CD", "iso3166_alpha3": "COD", "iso3166_num": "180", "iso3166_name": "Congo - Kinshasa", "name": "Congo - Kinshasa", "languages": ["fr-CD", "ln", "ktu", "kg", "sw", "lua"], "tld": ".cd", "capital": "Kinshasa", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "DK", "iso3166_alpha3": "DNK", "iso3166_num": "208", "iso3166_name": "Denmark", "name": "Denmark", "languages": ["da-DK", "en", "fo", "de-DK"], "tld": ".dk", "capital": "Copenhagen", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "DJ", "iso3166_alpha3": "DJI", "iso3166_num": "262", "iso3166_name": "Djibouti", "name": "Djibouti", "languages": ["fr-DJ", "ar", "so-DJ", "aa"], "tld": ".dj", "capital": "Djibouti", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "DM", "iso3166_alpha3": "DMA", "iso3166_num": "212", "iso3166_name": "Dominica", "name": "Dominica", "languages": ["en-DM"], "tld": ".dm", "capital": "Roseau", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "DO", "iso3166_alpha3": "DOM", "iso3166_num": "214", "iso3166_name": "Dominican Republic", "name": "Dominican Republic", "languages": ["es-DO"], "tld": ".do", "capital": "Santo Domingo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "EC", "iso3166_alpha3": "ECU", "iso3166_num": "218", "iso3166_name": "Ecuador", "name": "Ecuador", "languages": ["es-EC"], "tld": ".ec", "capital": "Quito", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "EG", "iso3166_alpha3": "EGY", "iso3166_num": "818", "iso3166_name": "Egypt", "name": "Egypt", "languages": ["ar-EG", "en", "fr"], "tld": ".eg", "capital": "Cairo", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "SV", "iso3166_alpha3": "SLV", "iso3166_num": "222", "iso3166_name": "El Salvador", "name": "El Salvador", "languages": ["es-SV"], "tld": ".sv", "capital": "San Salvador", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GQ", "iso3166_alpha3": "GNQ", "iso3166_num": "226", "iso3166_name": "Equatorial Guinea", "name": "Equatorial Guinea", "languages": ["es-GQ", "fr"], "tld": ".gq", "capital": "Malabo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ER", "iso3166_alpha3": "ERI", "iso3166_num": "232", "iso3166_name": "Eritrea", "name": "Eritrea", "languages": ["aa-ER", "ar", "tig", "kun", "ti-ER"], "tld": ".er", "capital": "Asmara", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "EE", "iso3166_alpha3": "EST", "iso3166_num": "233", "iso3166_name": "Estonia", "name": "Estonia", "languages": ["et", "ru"], "tld": ".ee", "capital": "Tallinn", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "ET", "iso3166_alpha3": "ETH", "iso3166_num": "231", "iso3166_name": "Ethiopia", "name": "Ethiopia", "languages": ["am", "en-ET", "om-ET", "ti-ET", "so-ET", "sid"], "tld": ".et", "capital": "Addis Ababa", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "FK", "iso3166_alpha3": "FLK", "iso3166_num": "238", "iso3166_name": "Falkland Islands", "name": "Falkland Islands", "languages": ["en-FK"], "tld": ".fk", "capital": "Stanley", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "FO", "iso3166_alpha3": "FRO", "iso3166_num": "234", "iso3166_name": "Faroe Islands", "name": "Faroe Islands", "languages": ["fo", "da-FO"], "tld": ".fo", "capital": "Torshavn", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "FJ", "iso3166_alpha3": "FJI", "iso3166_num": "242", "iso3166_name": "Fiji", "name": "Fiji", "languages": ["en-FJ", "fj"], "tld": ".fj", "capital": "Suva", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "FI", "iso3166_alpha3": "FIN", "iso3166_num": "246", "iso3166_name": "Finland", "name": "Finland", "languages": ["fi-FI", "sv-FI", "smn"], "tld": ".fi", "capital": "Helsinki", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "FR", "iso3166_alpha3": "FRA", "iso3166_num": "250", "iso3166_name": "France", "name": "France", "languages": ["fr-FR", "frp", "br", "co", "ca", "eu", "oc"], "tld": ".fr", "capital": "Paris", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "GF", "iso3166_alpha3": "GUF", "iso3166_num": "254", "iso3166_name": "French Guiana", "name": "French Guiana", "languages": ["fr-GF"], "tld": ".gf", "capital": "Cayenne", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PF", "iso3166_alpha3": "PYF", "iso3166_num": "258", "iso3166_name": "French Polynesia", "name": "French Polynesia", "languages": ["fr-PF", "ty"], "tld": ".pf", "capital": "Papeete", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TF", "iso3166_alpha3": "ATF", "iso3166_num": "260", "iso3166_name": "French Southern Territories", "name": "French Southern Territories", "languages": ["fr"], "tld": ".tf", "capital": "Port-aux-Francais", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GA", "iso3166_alpha3": "GAB", "iso3166_num": "266", "iso3166_name": "Gabon", "name": "Gabon", "languages": ["fr-GA"], "tld": ".ga", "capital": "Libreville", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GM", "iso3166_alpha3": "GMB", "iso3166_num": "270", "iso3166_name": "Gambia", "name": "Gambia", "languages": ["en-GM", "mnk", "wof", "wo", "ff"], "tld": ".gm", "capital": "Banjul", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GE", "iso3166_alpha3": "GEO", "iso3166_num": "268", "iso3166_name": "Georgia", "name": "Georgia", "languages": ["ka", "ru", "hy", "az"], "tld": ".ge", "capital": "Tbilisi", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "DE", "iso3166_alpha3": "DEU", "iso3166_num": "276", "iso3166_name": "Germany", "name": "Germany", "languages": ["de"], "tld": ".de", "capital": "Berlin", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "GH", "iso3166_alpha3": "GHA", "iso3166_num": "288", "iso3166_name": "Ghana", "name": "Ghana", "languages": ["en-GH", "ak", "ee", "tw"], "tld": ".gh", "capital": "Accra", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GI", "iso3166_alpha3": "GIB", "iso3166_num": "292", "iso3166_name": "Gibraltar", "name": "Gibraltar", "languages": ["en-GI", "es", "it", "pt"], "tld": ".gi", "capital": "Gibraltar", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "GR", "iso3166_alpha3": "GRC", "iso3166_num": "300", "iso3166_name": "Greece", "name": "Greece", "languages": ["el-GR", "en", "fr"], "tld": ".gr", "capital": "Athens", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "GL", "iso3166_alpha3": "GRL", "iso3166_num": "304", "iso3166_name": "Greenland", "name": "Greenland", "languages": ["kl", "da-GL", "en"], "tld": ".gl", "capital": "Nuuk", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "GD", "iso3166_alpha3": "GRD", "iso3166_num": "308", "iso3166_name": "Grenada", "name": "Grenada", "languages": ["en-GD"], "tld": ".gd", "capital": "St. George's", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GP", "iso3166_alpha3": "GLP", "iso3166_num": "312", "iso3166_name": "Guadeloupe", "name": "Guadeloupe", "languages": ["fr-GP"], "tld": ".gp", "capital": "Basse-Terre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GU", "iso3166_alpha3": "GUM", "iso3166_num": "316", "iso3166_name": "Guam", "name": "Guam", "languages": ["en-GU", "ch-GU"], "tld": ".gu", "capital": "Hagatna", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "GT", "iso3166_alpha3": "GTM", "iso3166_num": "320", "iso3166_name": "Guatemala", "name": "Guatemala", "languages": ["es-GT"], "tld": ".gt", "capital": "Guatemala City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "GG", "iso3166_alpha3": "GGY", "iso3166_num": "831", "iso3166_name": "Guernsey", "name": "Guernsey", "languages": ["en", "nrf"], "tld": ".gg", "capital": "St Peter Port", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "GN", "iso3166_alpha3": "GIN", "iso3166_num": "324", "iso3166_name": "Guinea", "name": "Guinea", "languages": ["fr-GN"], "tld": ".gn", "capital": "Conakry", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GW", "iso3166_alpha3": "GNB", "iso3166_num": "624", "iso3166_name": "Guinea-Bissau", "name": "Guinea-Bissau", "languages": ["pt-GW", "pov"], "tld": ".gw", "capital": "Bissau", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GY", "iso3166_alpha3": "GUY", "iso3166_num": "328", "iso3166_name": "Guyana", "name": "Guyana", "languages": ["en-GY"], "tld": ".gy", "capital": "Georgetown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HT", "iso3166_alpha3": "HTI", "iso3166_num": "332", "iso3166_name": "Haiti", "name": "Haiti", "languages": ["ht", "fr-HT"], "tld": ".ht", "capital": "Port-au-Prince", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HM", "iso3166_alpha3": "HMD", "iso3166_num": "334", "iso3166_name": "Heard & McDonald Islands", "name": "Heard & McDonald Islands", "languages": [""], "tld": ".hm", "capital": "", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "VA", "iso3166_alpha3": "VAT", "iso3166_num": "336", "iso3166_name": "Vatican City", "name": "Vatican City", "languages": ["la", "it", "fr"], "tld": ".va", "capital": "Vatican City", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "HN", "iso3166_alpha3": "HND", "iso3166_num": "340", "iso3166_name": "Honduras", "name": "Honduras", "languages": ["es-HN", "cab", "miq"], "tld": ".hn", "capital": "Tegucigalpa", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "HU", "iso3166_alpha3": "HUN", "iso3166_num": "348", "iso3166_name": "Hungary", "name": "Hungary", "languages": ["hu-HU"], "tld": ".hu", "capital": "Budapest", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "IS", "iso3166_alpha3": "ISL", "iso3166_num": "352", "iso3166_name": "Iceland", "name": "Iceland", "languages": ["is", "en", "de", "da", "sv", "no"], "tld": ".is", "capital": "Reykjavik", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IN", "iso3166_alpha3": "IND", "iso3166_num": "356", "iso3166_name": "India", "name": "India", "languages": ["en-IN", "hi", "bn", "te", "mr", "ta", "ur", "gu", "kn", "ml", "or", "pa", "as", "bh", "sat", "ks", "ne", "sd", "kok", "doi", "mni", "sit", "sa", "fr", "lus", "inc"], "tld": ".in", "capital": "New Delhi", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "ID", "iso3166_alpha3": "IDN", "iso3166_num": "360", "iso3166_name": "Indonesia", "name": "Indonesia", "languages": ["id", "en", "nl", "jv"], "tld": ".id", "capital": "Jakarta", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "IR", "iso3166_alpha3": "IRN", "iso3166_num": "364", "iso3166_name": "Iran", "name": "Iran", "languages": ["fa-IR", "ku"], "tld": ".ir", "capital": "Tehran", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "IQ", "iso3166_alpha3": "IRQ", "iso3166_num": "368", "iso3166_name": "Iraq", "name": "Iraq", "languages": ["ar-IQ", "ku", "hy"], "tld": ".iq", "capital": "Baghdad", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "IE", "iso3166_alpha3": "IRL", "iso3166_num": "372", "iso3166_name": "Ireland", "name": "Ireland", "languages": ["en-IE", "ga-IE"], "tld": ".ie", "capital": "Dublin", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IM", "iso3166_alpha3": "IMN", "iso3166_num": "833", "iso3166_name": "Isle of Man", "name": "Isle of Man", "languages": ["en", "gv"], "tld": ".im", "capital": "Douglas", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "IL", "iso3166_alpha3": "ISR", "iso3166_num": "376", "iso3166_name": "Israel", "name": "Israel", "languages": ["he", "ar-IL", "en-IL", ""], "tld": ".il", "capital": "Jerusalem", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "IT", "iso3166_alpha3": "ITA", "iso3166_num": "380", "iso3166_name": "Italy", "name": "Italy", "languages": ["it-IT", "de-IT", "fr-IT", "sc", "ca", "co", "sl"], "tld": ".it", "capital": "Rome", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "JM", "iso3166_alpha3": "JAM", "iso3166_num": "388", "iso3166_name": "Jamaica", "name": "Jamaica", "languages": ["en-JM"], "tld": ".jm", "capital": "Kingston", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "JP", "iso3166_alpha3": "JPN", "iso3166_num": "392", "iso3166_name": "Japan", "name": "Japan", "languages": ["ja"], "tld": ".jp", "capital": "Tokyo", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "JE", "iso3166_alpha3": "JEY", "iso3166_num": "832", "iso3166_name": "Jersey", "name": "Jersey", "languages": ["en", "fr", "nrf"], "tld": ".je", "capital": "Saint Helier", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "JO", "iso3166_alpha3": "JOR", "iso3166_num": "400", "iso3166_name": "Jordan", "name": "Jordan", "languages": ["ar-JO", "en"], "tld": ".jo", "capital": "Amman", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KZ", "iso3166_alpha3": "KAZ", "iso3166_num": "398", "iso3166_name": "Kazakhstan", "name": "Kazakhstan", "languages": ["kk", "ru"], "tld": ".kz", "capital": "Nur-Sultan", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "KE", "iso3166_alpha3": "KEN", "iso3166_num": "404", "iso3166_name": "Kenya", "name": "Kenya", "languages": ["en-KE", "sw-KE"], "tld": ".ke", "capital": "Nairobi", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KI", "iso3166_alpha3": "KIR", "iso3166_num": "296", "iso3166_name": "Kiribati", "name": "Kiribati", "languages": ["en-KI", "gil"], "tld": ".ki", "capital": "Tarawa", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "KW", "iso3166_alpha3": "KWT", "iso3166_num": "414", "iso3166_name": "Kuwait", "name": "Kuwait", "languages": ["ar-KW", "en"], "tld": ".kw", "capital": "Kuwait City", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KG", "iso3166_alpha3": "KGZ", "iso3166_num": "417", "iso3166_name": "Kyrgyzstan", "name": "Kyrgyzstan", "languages": ["ky", "uz", "ru"], "tld": ".kg", "capital": "Bishkek", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "LA", "iso3166_alpha3": "LAO", "iso3166_num": "418", "iso3166_name": "Laos", "name": "Laos", "languages": ["lo", "fr", "en"], "tld": ".la", "capital": "Vientiane", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "LV", "iso3166_alpha3": "LVA", "iso3166_num": "428", "iso3166_name": "Latvia", "name": "Latvia", "languages": ["lv", "ru", "lt"], "tld": ".lv", "capital": "Riga", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "LB", "iso3166_alpha3": "LBN", "iso3166_num": "422", "iso3166_name": "Lebanon", "name": "Lebanon", "languages": ["ar-LB", "fr-LB", "en", "hy"], "tld": ".lb", "capital": "Beirut", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "LS", "iso3166_alpha3": "LSO", "iso3166_num": "426", "iso3166_name": "Lesotho", "name": "Lesotho", "languages": ["en-LS", "st", "zu", "xh"], "tld": ".ls", "capital": "Maseru", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "LR", "iso3166_alpha3": "LBR", "iso3166_num": "430", "iso3166_name": "Liberia", "name": "Liberia", "languages": ["en-LR"], "tld": ".lr", "capital": "Monrovia", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "LY", "iso3166_alpha3": "LBY", "iso3166_num": "434", "iso3166_name": "Libya", "name": "Libya", "languages": ["ar-LY", "it", "en"], "tld": ".ly", "capital": "Tripoli", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "LI", "iso3166_alpha3": "LIE", "iso3166_num": "438", "iso3166_name": "Liechtenstein", "name": "Liechtenstein", "languages": ["de-LI"], "tld": ".li", "capital": "Vaduz", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "LT", "iso3166_alpha3": "LTU", "iso3166_num": "440", "iso3166_name": "Lithuania", "name": "Lithuania", "languages": ["lt", "ru", "pl"], "tld": ".lt", "capital": "Vilnius", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "LU", "iso3166_alpha3": "LUX", "iso3166_num": "442", "iso3166_name": "Luxembourg", "name": "Luxembourg", "languages": ["lb", "de-LU", "fr-LU"], "tld": ".lu", "capital": "Luxembourg", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "MG", "iso3166_alpha3": "MDG", "iso3166_num": "450", "iso3166_name": "Madagascar", "name": "Madagascar", "languages": ["fr-MG", "mg"], "tld": ".mg", "capital": "Antananarivo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MW", "iso3166_alpha3": "MWI", "iso3166_num": "454", "iso3166_name": "Malawi", "name": "Malawi", "languages": ["ny", "yao", "tum", "swk"], "tld": ".mw", "capital": "Lilongwe", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MY", "iso3166_alpha3": "MYS", "iso3166_num": "458", "iso3166_name": "Malaysia", "name": "Malaysia", "languages": ["ms-MY", "en", "zh", "ta", "te", "ml", "pa", "th"], "tld": ".my", "capital": "Kuala Lumpur", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "MV", "iso3166_alpha3": "MDV", "iso3166_num": "462", "iso3166_name": "Maldives", "name": "Maldives", "languages": ["dv", "en"], "tld": ".mv", "capital": "Male", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "ML", "iso3166_alpha3": "MLI", "iso3166_num": "466", "iso3166_name": "Mali", "name": "Mali", "languages": ["fr-ML", "bm"], "tld": ".ml", "capital": "Bamako", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MT", "iso3166_alpha3": "MLT", "iso3166_num": "470", "iso3166_name": "Malta", "name": "Malta", "languages": ["mt", "en-MT"], "tld": ".mt", "capital": "Valletta", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "MH", "iso3166_alpha3": "MHL", "iso3166_num": "584", "iso3166_name": "Marshall Islands", "name": "Marshall Islands", "languages": ["mh", "en-MH"], "tld": ".mh", "capital": "Majuro", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "MQ", "iso3166_alpha3": "MTQ", "iso3166_num": "474", "iso3166_name": "Martinique", "name": "Martinique", "languages": ["fr-MQ"], "tld": ".mq", "capital": "Fort-de-France", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MR", "iso3166_alpha3": "MRT", "iso3166_num": "478", "iso3166_name": "Mauritania", "name": "Mauritania", "languages": ["ar-MR", "fuc", "snk", "fr", "mey", "wo"], "tld": ".mr", "capital": "Nouakchott", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MU", "iso3166_alpha3": "MUS", "iso3166_num": "480", "iso3166_name": "Mauritius", "name": "Mauritius", "languages": ["en-MU", "bho", "fr"], "tld": ".mu", "capital": "Port Louis", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "YT", "iso3166_alpha3": "MYT", "iso3166_num": "175", "iso3166_name": "Mayotte", "name": "Mayotte", "languages": ["fr-YT"], "tld": ".yt", "capital": "Mamoudzou", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MX", "iso3166_alpha3": "MEX", "iso3166_num": "484", "iso3166_name": "Mexico", "name": "Mexico", "languages": ["es-MX"], "tld": ".mx", "capital": "Mexico City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "FM", "iso3166_alpha3": "FSM", "iso3166_num": "583", "iso3166_name": "Micronesia", "name": "Micronesia", "languages": ["en-FM", "chk", "pon", "yap", "kos", "uli", "woe", "nkr", "kpg"], "tld": ".fm", "capital": "Palikir", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "MC", "iso3166_alpha3": "MCO", "iso3166_num": "492", "iso3166_name": "Monaco", "name": "Monaco", "languages": ["fr-MC", "en", "it"], "tld": ".mc", "capital": "Monaco", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "MN", "iso3166_alpha3": "MNG", "iso3166_num": "496", "iso3166_name": "Mongolia", "name": "Mongolia", "languages": ["mn", "ru"], "tld": ".mn", "capital": "Ulaanbaatar", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "ME", "iso3166_alpha3": "MNE", "iso3166_num": "499", "iso3166_name": "Montenegro", "name": "Montenegro", "languages": ["sr", "hu", "bs", "sq", "hr", "rom"], "tld": ".me", "capital": "Podgorica", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "MS", "iso3166_alpha3": "MSR", "iso3166_num": "500", "iso3166_name": "Montserrat", "name": "Montserrat", "languages": ["en-MS"], "tld": ".ms", "capital": "Plymouth", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MA", "iso3166_alpha3": "MAR", "iso3166_num": "504", "iso3166_name": "Morocco", "name": "Morocco", "languages": ["ar-MA", "ber", "fr"], "tld": ".ma", "capital": "Rabat", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "MZ", "iso3166_alpha3": "MOZ", "iso3166_num": "508", "iso3166_name": "Mozambique", "name": "Mozambique", "languages": ["pt-MZ", "vmw"], "tld": ".mz", "capital": "Maputo", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "MM", "iso3166_alpha3": "MMR", "iso3166_num": "104", "iso3166_name": "Myanmar", "name": "Myanmar", "languages": ["my"], "tld": ".mm", "capital": "Nay Pyi Taw", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "NA", "iso3166_alpha3": "NAM", "iso3166_num": "516", "iso3166_name": "Namibia", "name": "Namibia", "languages": ["en-NA", "af", "de", "hz", "naq"], "tld": ".na", "capital": "Windhoek", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NR", "iso3166_alpha3": "NRU", "iso3166_num": "520", "iso3166_name": "Nauru", "name": "Nauru", "languages": ["na", "en-NR"], "tld": ".nr", "capital": "Yaren", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "NP", "iso3166_alpha3": "NPL", "iso3166_num": "524", "iso3166_name": "Nepal", "name": "Nepal", "languages": ["ne", "en"], "tld": ".np", "capital": "Kathmandu", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "NL", "iso3166_alpha3": "NLD", "iso3166_num": "528", "iso3166_name": "Netherlands", "name": "Netherlands", "languages": ["nl-NL", "fy-NL"], "tld": ".nl", "capital": "Amsterdam", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "NC", "iso3166_alpha3": "NCL", "iso3166_num": "540", "iso3166_name": "New Caledonia", "name": "New Caledonia", "languages": ["fr-NC"], "tld": ".nc", "capital": "Noumea", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "NZ", "iso3166_alpha3": "NZL", "iso3166_num": "554", "iso3166_name": "New Zealand", "name": "New Zealand", "languages": ["en-NZ", "mi"], "tld": ".nz", "capital": "Wellington", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "NI", "iso3166_alpha3": "NIC", "iso3166_num": "558", "iso3166_name": "Nicaragua", "name": "Nicaragua", "languages": ["es-NI", "en"], "tld": ".ni", "capital": "Managua", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "NE", "iso3166_alpha3": "NER", "iso3166_num": "562", "iso3166_name": "Niger", "name": "Niger", "languages": ["fr-NE", "ha", "kr", "dje"], "tld": ".ne", "capital": "Niamey", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NG", "iso3166_alpha3": "NGA", "iso3166_num": "566", "iso3166_name": "Nigeria", "name": "Nigeria", "languages": ["en-NG", "ha", "yo", "ig", "ff"], "tld": ".ng", "capital": "Abuja", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "NU", "iso3166_alpha3": "NIU", "iso3166_num": "570", "iso3166_name": "Niue", "name": "Niue", "languages": ["niu", "en-NU"], "tld": ".nu", "capital": "Alofi", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "NF", "iso3166_alpha3": "NFK", "iso3166_num": "574", "iso3166_name": "Norfolk Island", "name": "Norfolk Island", "languages": ["en-NF"], "tld": ".nf", "capital": "Kingston", "region_code": "009", "sub_region_code": "053"}, {"iso3166_alpha2": "MP", "iso3166_alpha3": "MNP", "iso3166_num": "580", "iso3166_name": "Northern Mariana Islands", "name": "Northern Mariana Islands", "languages": ["fil", "tl", "zh", "ch-MP", "en-MP"], "tld": ".mp", "capital": "Saipan", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "NO", "iso3166_alpha3": "NOR", "iso3166_num": "578", "iso3166_name": "Norway", "name": "Norway", "languages": ["no", "nb", "nn", "se", "fi"], "tld": ".no", "capital": "Oslo", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "OM", "iso3166_alpha3": "OMN", "iso3166_num": "512", "iso3166_name": "Oman", "name": "Oman", "languages": ["ar-OM", "en", "bal", "ur"], "tld": ".om", "capital": "Muscat", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "PK", "iso3166_alpha3": "PAK", "iso3166_num": "586", "iso3166_name": "Pakistan", "name": "Pakistan", "languages": ["ur-PK", "en-PK", "pa", "sd", "ps", "brh"], "tld": ".pk", "capital": "Islamabad", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "PW", "iso3166_alpha3": "PLW", "iso3166_num": "585", "iso3166_name": "Palau", "name": "Palau", "languages": ["pau", "sov", "en-PW", "tox", "ja", "fil", "zh"], "tld": ".pw", "capital": "Melekeok", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "PA", "iso3166_alpha3": "PAN", "iso3166_num": "591", "iso3166_name": "Panama", "name": "Panama", "languages": ["es-PA", "en"], "tld": ".pa", "capital": "Panama City", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PG", "iso3166_alpha3": "PNG", "iso3166_num": "598", "iso3166_name": "Papua New Guinea", "name": "Papua New Guinea", "languages": ["en-PG", "ho", "meu", "tpi"], "tld": ".pg", "capital": "Port Moresby", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "PY", "iso3166_alpha3": "PRY", "iso3166_num": "600", "iso3166_name": "Paraguay", "name": "Paraguay", "languages": ["es-PY", "gn"], "tld": ".py", "capital": "Asuncion", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PE", "iso3166_alpha3": "PER", "iso3166_num": "604", "iso3166_name": "Peru", "name": "Peru", "languages": ["es-PE", "qu", "ay"], "tld": ".pe", "capital": "Lima", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PH", "iso3166_alpha3": "PHL", "iso3166_num": "608", "iso3166_name": "Philippines", "name": "Philippines", "languages": ["tl", "en-PH", "fil", "ceb", "tgl", "ilo", "hil", "war", "pam", "bik", "bcl", "pag", "mrw", "tsg", "mdh", "cbk", "krj", "sgd", "msb", "akl", "ibg", "yka", "mta", "abx"], "tld": ".ph", "capital": "Manila", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "PN", "iso3166_alpha3": "PCN", "iso3166_num": "612", "iso3166_name": "Pitcairn Islands", "name": "Pitcairn Islands", "languages": ["en-PN"], "tld": ".pn", "capital": "Adamstown", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "PL", "iso3166_alpha3": "POL", "iso3166_num": "616", "iso3166_name": "Poland", "name": "Poland", "languages": ["pl"], "tld": ".pl", "capital": "Warsaw", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "PT", "iso3166_alpha3": "PRT", "iso3166_num": "620", "iso3166_name": "Portugal", "name": "Portugal", "languages": ["pt-PT", "mwl"], "tld": ".pt", "capital": "Lisbon", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "PR", "iso3166_alpha3": "PRI", "iso3166_num": "630", "iso3166_name": "Puerto Rico", "name": "Puerto Rico", "languages": ["en-PR", "es-PR"], "tld": ".pr", "capital": "San Juan", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "QA", "iso3166_alpha3": "QAT", "iso3166_num": "634", "iso3166_name": "Qatar", "name": "Qatar", "languages": ["ar-QA", "es"], "tld": ".qa", "capital": "Doha", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "KR", "iso3166_alpha3": "KOR", "iso3166_num": "410", "iso3166_name": "South Korea", "name": "South Korea", "languages": ["ko-KR", "en"], "tld": ".kr", "capital": "Seoul", "region_code": "142", "sub_region_code": "030"}, {"iso3166_alpha2": "MD", "iso3166_alpha3": "MDA", "iso3166_num": "498", "iso3166_name": "Moldova", "name": "Moldova", "languages": ["ro", "ru", "gag", "tr"], "tld": ".md", "capital": "Chisinau", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RO", "iso3166_alpha3": "ROU", "iso3166_num": "642", "iso3166_name": "Romania", "name": "Romania", "languages": ["ro", "hu", "rom"], "tld": ".ro", "capital": "Bucharest", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RU", "iso3166_alpha3": "RUS", "iso3166_num": "643", "iso3166_name": "Russia", "name": "Russia", "languages": ["ru", "tt", "xal", "cau", "ady", "kv", "ce", "tyv", "cv", "udm", "tut", "mns", "bua", "myv", "mdf", "chm", "ba", "inh", "tut", "kbd", "krc", "av", "sah", "nog"], "tld": ".ru", "capital": "Moscow", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "RW", "iso3166_alpha3": "RWA", "iso3166_num": "646", "iso3166_name": "Rwanda", "name": "Rwanda", "languages": ["rw", "en-RW", "fr-RW", "sw"], "tld": ".rw", "capital": "Kigali", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "RE", "iso3166_alpha3": "REU", "iso3166_num": "638", "iso3166_name": "R\u00e9union", "name": "R\u00e9union", "languages": ["fr-RE"], "tld": ".re", "capital": "Saint-Denis", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "BL", "iso3166_alpha3": "BLM", "iso3166_num": "652", "iso3166_name": "St. Barth\u00e9lemy", "name": "St. Barth\u00e9lemy", "languages": ["fr"], "tld": ".gp", "capital": "Gustavia", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SH", "iso3166_alpha3": "SHN", "iso3166_num": "654", "iso3166_name": "St. Helena", "name": "St. Helena", "languages": ["en-SH"], "tld": ".sh", "capital": "Jamestown", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "KN", "iso3166_alpha3": "KNA", "iso3166_num": "659", "iso3166_name": "St. Kitts & Nevis", "name": "St. Kitts & Nevis", "languages": ["en-KN"], "tld": ".kn", "capital": "Basseterre", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "LC", "iso3166_alpha3": "LCA", "iso3166_num": "662", "iso3166_name": "St. Lucia", "name": "St. Lucia", "languages": ["en-LC"], "tld": ".lc", "capital": "Castries", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "MF", "iso3166_alpha3": "MAF", "iso3166_num": "663", "iso3166_name": "St. Martin", "name": "St. Martin", "languages": ["fr"], "tld": ".gp", "capital": "Marigot", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "PM", "iso3166_alpha3": "SPM", "iso3166_num": "666", "iso3166_name": "St. Pierre & Miquelon", "name": "St. Pierre & Miquelon", "languages": ["fr-PM"], "tld": ".pm", "capital": "Saint-Pierre", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "VC", "iso3166_alpha3": "VCT", "iso3166_num": "670", "iso3166_name": "St. Vincent & Grenadines", "name": "St. Vincent & Grenadines", "languages": ["en-VC", "fr"], "tld": ".vc", "capital": "Kingstown", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "WS", "iso3166_alpha3": "WSM", "iso3166_num": "882", "iso3166_name": "Samoa", "name": "Samoa", "languages": ["sm", "en-WS"], "tld": ".ws", "capital": "Apia", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "SM", "iso3166_alpha3": "SMR", "iso3166_num": "674", "iso3166_name": "San Marino", "name": "San Marino", "languages": ["it-SM"], "tld": ".sm", "capital": "San Marino", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "ST", "iso3166_alpha3": "STP", "iso3166_num": "678", "iso3166_name": "S\u00e3o Tom\u00e9 & Pr\u00edncipe", "name": "S\u00e3o Tom\u00e9 & Pr\u00edncipe", "languages": ["pt-ST"], "tld": ".st", "capital": "Sao Tome", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SA", "iso3166_alpha3": "SAU", "iso3166_num": "682", "iso3166_name": "Saudi Arabia", "name": "Saudi Arabia", "languages": ["ar-SA"], "tld": ".sa", "capital": "Riyadh", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "SN", "iso3166_alpha3": "SEN", "iso3166_num": "686", "iso3166_name": "Senegal", "name": "Senegal", "languages": ["fr-SN", "wo", "fuc", "mnk"], "tld": ".sn", "capital": "Dakar", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "RS", "iso3166_alpha3": "SRB", "iso3166_num": "688", "iso3166_name": "Serbia", "name": "Serbia", "languages": ["sr", "hu", "bs", "rom"], "tld": ".rs", "capital": "Belgrade", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "SC", "iso3166_alpha3": "SYC", "iso3166_num": "690", "iso3166_name": "Seychelles", "name": "Seychelles", "languages": ["en-SC", "fr-SC"], "tld": ".sc", "capital": "Victoria", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SL", "iso3166_alpha3": "SLE", "iso3166_num": "694", "iso3166_name": "Sierra Leone", "name": "Sierra Leone", "languages": ["en-SL", "men", "tem"], "tld": ".sl", "capital": "Freetown", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SG", "iso3166_alpha3": "SGP", "iso3166_num": "702", "iso3166_name": "Singapore", "name": "Singapore", "languages": ["cmn", "en-SG", "ms-SG", "ta-SG", "zh-SG"], "tld": ".sg", "capital": "Singapore", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "SX", "iso3166_alpha3": "SXM", "iso3166_num": "534", "iso3166_name": "Sint Maarten", "name": "Sint Maarten", "languages": ["nl", "en"], "tld": ".sx", "capital": "Philipsburg", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SK", "iso3166_alpha3": "SVK", "iso3166_num": "703", "iso3166_name": "Slovakia", "name": "Slovakia", "languages": ["sk", "hu"], "tld": ".sk", "capital": "Bratislava", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "SI", "iso3166_alpha3": "SVN", "iso3166_num": "705", "iso3166_name": "Slovenia", "name": "Slovenia", "languages": ["sl", "sh"], "tld": ".si", "capital": "Ljubljana", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "SB", "iso3166_alpha3": "SLB", "iso3166_num": "090", "iso3166_name": "Solomon Islands", "name": "Solomon Islands", "languages": ["en-SB", "tpi"], "tld": ".sb", "capital": "Honiara", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "SO", "iso3166_alpha3": "SOM", "iso3166_num": "706", "iso3166_name": "Somalia", "name": "Somalia", "languages": ["so-SO", "ar-SO", "it", "en-SO"], "tld": ".so", "capital": "Mogadishu", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ZA", "iso3166_alpha3": "ZAF", "iso3166_num": "710", "iso3166_name": "South Africa", "name": "South Africa", "languages": ["zu", "xh", "af", "nso", "en-ZA", "tn", "st", "ts", "ss", "ve", "nr"], "tld": ".za", "capital": "Pretoria", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "GS", "iso3166_alpha3": "SGS", "iso3166_num": "239", "iso3166_name": "South Georgia & South Sandwich Islands", "name": "South Georgia & South Sandwich Islands", "languages": ["en"], "tld": ".gs", "capital": "Grytviken", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SS", "iso3166_alpha3": "SSD", "iso3166_num": "728", "iso3166_name": "South Sudan", "name": "South Sudan", "languages": ["en"], "tld": "", "capital": "Juba", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ES", "iso3166_alpha3": "ESP", "iso3166_num": "724", "iso3166_name": "Spain", "name": "Spain", "languages": ["es-ES", "ca", "gl", "eu", "oc"], "tld": ".es", "capital": "Madrid", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "LK", "iso3166_alpha3": "LKA", "iso3166_num": "144", "iso3166_name": "Sri Lanka", "name": "Sri Lanka", "languages": ["si", "ta", "en"], "tld": ".lk", "capital": "Colombo", "region_code": "142", "sub_region_code": "034"}, {"iso3166_alpha2": "PS", "iso3166_alpha3": "PSE", "iso3166_num": "275", "iso3166_name": "Palestine", "name": "Palestine", "languages": ["ar-PS"], "tld": ".ps", "capital": "East Jerusalem", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "SD", "iso3166_alpha3": "SDN", "iso3166_num": "729", "iso3166_name": "Sudan", "name": "Sudan", "languages": ["ar-SD", "en", "fia"], "tld": ".sd", "capital": "Khartoum", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "SR", "iso3166_alpha3": "SUR", "iso3166_num": "740", "iso3166_name": "Suriname", "name": "Suriname", "languages": ["nl-SR", "en", "srn", "hns", "jv"], "tld": ".sr", "capital": "Paramaribo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "SJ", "iso3166_alpha3": "SJM", "iso3166_num": "744", "iso3166_name": "Svalbard & Jan Mayen", "name": "Svalbard & Jan Mayen", "languages": ["no", "ru"], "tld": ".sj", "capital": "Longyearbyen", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "SZ", "iso3166_alpha3": "SWZ", "iso3166_num": "748", "iso3166_name": "Swaziland", "name": "Eswatini", "languages": ["en-SZ", "ss-SZ"], "tld": ".sz", "capital": "Mbabane", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "SE", "iso3166_alpha3": "SWE", "iso3166_num": "752", "iso3166_name": "Sweden", "name": "Sweden", "languages": ["sv-SE", "se", "sma", "fi-SE"], "tld": ".se", "capital": "Stockholm", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "CH", "iso3166_alpha3": "CHE", "iso3166_num": "756", "iso3166_name": "Switzerland", "name": "Switzerland", "languages": ["de-CH", "fr-CH", "it-CH", "rm"], "tld": ".ch", "capital": "Bern", "region_code": "150", "sub_region_code": "155"}, {"iso3166_alpha2": "SY", "iso3166_alpha3": "SYR", "iso3166_num": "760", "iso3166_name": "Syria", "name": "Syria", "languages": ["ar-SY", "ku", "hy", "arc", "fr", "en"], "tld": ".sy", "capital": "Damascus", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "TJ", "iso3166_alpha3": "TJK", "iso3166_num": "762", "iso3166_name": "Tajikistan", "name": "Tajikistan", "languages": ["tg", "ru"], "tld": ".tj", "capital": "Dushanbe", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "TH", "iso3166_alpha3": "THA", "iso3166_num": "764", "iso3166_name": "Thailand", "name": "Thailand", "languages": ["th", "en"], "tld": ".th", "capital": "Bangkok", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "MK", "iso3166_alpha3": "MKD", "iso3166_num": "807", "iso3166_name": "Macedonia", "name": "North Macedonia", "languages": ["mk", "sq", "tr", "rmm", "sr"], "tld": ".mk", "capital": "Skopje", "region_code": "150", "sub_region_code": "039"}, {"iso3166_alpha2": "TL", "iso3166_alpha3": "TLS", "iso3166_num": "626", "iso3166_name": "Timor-Leste", "name": "Timor-Leste", "languages": ["tet", "pt-TL", "id", "en"], "tld": ".tl", "capital": "Dili", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "TG", "iso3166_alpha3": "TGO", "iso3166_num": "768", "iso3166_name": "Togo", "name": "Togo", "languages": ["fr-TG", "ee", "hna", "kbp", "dag", "ha"], "tld": ".tg", "capital": "Lome", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "TK", "iso3166_alpha3": "TKL", "iso3166_num": "772", "iso3166_name": "Tokelau", "name": "Tokelau", "languages": ["tkl", "en-TK"], "tld": ".tk", "capital": "", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TO", "iso3166_alpha3": "TON", "iso3166_num": "776", "iso3166_name": "Tonga", "name": "Tonga", "languages": ["to", "en-TO"], "tld": ".to", "capital": "Nuku'alofa", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "TT", "iso3166_alpha3": "TTO", "iso3166_num": "780", "iso3166_name": "Trinidad & Tobago", "name": "Trinidad & Tobago", "languages": ["en-TT", "hns", "fr", "es", "zh"], "tld": ".tt", "capital": "Port of Spain", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "TN", "iso3166_alpha3": "TUN", "iso3166_num": "788", "iso3166_name": "Tunisia", "name": "Tunisia", "languages": ["ar-TN", "fr"], "tld": ".tn", "capital": "Tunis", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "TR", "iso3166_alpha3": "TUR", "iso3166_num": "792", "iso3166_name": "Turkey", "name": "Turkey", "languages": ["tr-TR", "ku", "diq", "az", "av"], "tld": ".tr", "capital": "Ankara", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "TM", "iso3166_alpha3": "TKM", "iso3166_num": "795", "iso3166_name": "Turkmenistan", "name": "Turkmenistan", "languages": ["tk", "ru", "uz"], "tld": ".tm", "capital": "Ashgabat", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "TC", "iso3166_alpha3": "TCA", "iso3166_num": "796", "iso3166_name": "Turks & Caicos Islands", "name": "Turks & Caicos Islands", "languages": ["en-TC"], "tld": ".tc", "capital": "Cockburn Town", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "TV", "iso3166_alpha3": "TUV", "iso3166_num": "798", "iso3166_name": "Tuvalu", "name": "Tuvalu", "languages": ["tvl", "en", "sm", "gil"], "tld": ".tv", "capital": "Funafuti", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "UG", "iso3166_alpha3": "UGA", "iso3166_num": "800", "iso3166_name": "Uganda", "name": "Uganda", "languages": ["en-UG", "lg", "sw", "ar"], "tld": ".ug", "capital": "Kampala", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "UA", "iso3166_alpha3": "UKR", "iso3166_num": "804", "iso3166_name": "Ukraine", "name": "Ukraine", "languages": ["uk", "ru-UA", "rom", "pl", "hu"], "tld": ".ua", "capital": "Kyiv", "region_code": "150", "sub_region_code": "151"}, {"iso3166_alpha2": "AE", "iso3166_alpha3": "ARE", "iso3166_num": "784", "iso3166_name": "United Arab Emirates", "name": "United Arab Emirates", "languages": ["ar-AE", "fa", "en", "hi", "ur"], "tld": ".ae", "capital": "Abu Dhabi", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "GB", "iso3166_alpha3": "GBR", "iso3166_num": "826", "iso3166_name": "UK", "name": "United Kingdom", "languages": ["en-GB", "cy-GB", "gd"], "tld": ".uk", "capital": "London", "region_code": "150", "sub_region_code": "154"}, {"iso3166_alpha2": "TZ", "iso3166_alpha3": "TZA", "iso3166_num": "834", "iso3166_name": "Tanzania", "name": "Tanzania", "languages": ["sw-TZ", "en", "ar"], "tld": ".tz", "capital": "Dodoma", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "UM", "iso3166_alpha3": "UMI", "iso3166_num": "581", "iso3166_name": "U.S. Outlying Islands", "name": "U.S. Outlying Islands", "languages": ["en-UM"], "tld": ".um", "capital": "", "region_code": "009", "sub_region_code": "057"}, {"iso3166_alpha2": "VI", "iso3166_alpha3": "VIR", "iso3166_num": "850", "iso3166_name": "U.S. Virgin Islands", "name": "U.S. Virgin Islands", "languages": ["en-VI"], "tld": ".vi", "capital": "Charlotte Amalie", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "US", "iso3166_alpha3": "USA", "iso3166_num": "840", "iso3166_name": "US", "name": "United States", "languages": ["en-US", "es-US", "haw", "fr"], "tld": ".us", "capital": "Washington", "region_code": "019", "sub_region_code": "021"}, {"iso3166_alpha2": "UY", "iso3166_alpha3": "URY", "iso3166_num": "858", "iso3166_name": "Uruguay", "name": "Uruguay", "languages": ["es-UY"], "tld": ".uy", "capital": "Montevideo", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "UZ", "iso3166_alpha3": "UZB", "iso3166_num": "860", "iso3166_name": "Uzbekistan", "name": "Uzbekistan", "languages": ["uz", "ru", "tg"], "tld": ".uz", "capital": "Tashkent", "region_code": "142", "sub_region_code": "143"}, {"iso3166_alpha2": "VU", "iso3166_alpha3": "VUT", "iso3166_num": "548", "iso3166_name": "Vanuatu", "name": "Vanuatu", "languages": ["bi", "en-VU", "fr-VU"], "tld": ".vu", "capital": "Port Vila", "region_code": "009", "sub_region_code": "054"}, {"iso3166_alpha2": "VE", "iso3166_alpha3": "VEN", "iso3166_num": "862", "iso3166_name": "Venezuela", "name": "Venezuela", "languages": ["es-VE"], "tld": ".ve", "capital": "Caracas", "region_code": "019", "sub_region_code": "419"}, {"iso3166_alpha2": "VN", "iso3166_alpha3": "VNM", "iso3166_num": "704", "iso3166_name": "Vietnam", "name": "Vietnam", "languages": ["vi", "en", "fr", "zh", "km"], "tld": ".vn", "capital": "Hanoi", "region_code": "142", "sub_region_code": "035"}, {"iso3166_alpha2": "WF", "iso3166_alpha3": "WLF", "iso3166_num": "876", "iso3166_name": "Wallis & Futuna", "name": "Wallis & Futuna", "languages": ["wls", "fud", "fr-WF"], "tld": ".wf", "capital": "Mata Utu", "region_code": "009", "sub_region_code": "061"}, {"iso3166_alpha2": "EH", "iso3166_alpha3": "ESH", "iso3166_num": "732", "iso3166_name": "Western Sahara", "name": "Western Sahara", "languages": ["ar", "mey"], "tld": ".eh", "capital": "El-Aaiun", "region_code": "002", "sub_region_code": "015"}, {"iso3166_alpha2": "YE", "iso3166_alpha3": "YEM", "iso3166_num": "887", "iso3166_name": "Yemen", "name": "Yemen", "languages": ["ar-YE"], "tld": ".ye", "capital": "Sanaa", "region_code": "142", "sub_region_code": "145"}, {"iso3166_alpha2": "ZM", "iso3166_alpha3": "ZMB", "iso3166_num": "894", "iso3166_name": "Zambia", "name": "Zambia", "languages": ["en-ZM", "bem", "loz", "lun", "lue", "ny", "toi"], "tld": ".zm", "capital": "Lusaka", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "ZW", "iso3166_alpha3": "ZWE", "iso3166_num": "716", "iso3166_name": "Zimbabwe", "name": "Zimbabwe", "languages": ["en-ZW", "sn", "nr", "nd"], "tld": ".zw", "capital": "Harare", "region_code": "002", "sub_region_code": "202"}, {"iso3166_alpha2": "AX", "iso3166_alpha3": "ALA", "iso3166_num": "248", "iso3166_name": "\u00c5land Islands", "name": "\u00c5land Islands", "languages": ["sv-AX"], "tld": ".ax", "capital": "Mariehamn", "region_code": "150", "sub_region_code": "154"}] diff --git a/detector/detector/detector.py b/detector/detector/detector.py new file mode 100755 index 00000000..46a2a2b6 --- /dev/null +++ b/detector/detector/detector.py @@ -0,0 +1,926 @@ +#!/usr/bin/env python3 +# # -*- coding: utf-8 -*- + +""" +OONI Event detector + +Run sequence: + +Fetch already-processed mean values from internal data directory. +This is done to speed up restarts as processing historical data from the database +would take a very long time. + +Fetch historical msmt from the fastpath and measurement/report/input tables + +Fetch realtime msmt by subscribing to the notifications channel `fastpath` + +Analise msmt score with moving average to detect blocking/unblocking + +Save outputs to local directories: + - RSS feed /var/lib/detector/rss/ + rss/global.xml All events, globally + rss/by-country/.xml Events by country + rss/type-inp/.xml Events by test_name and input + rss/cc-type-inp/.xml Events by CC, test_name and input + - JSON files with block/unblock events /var/lib/detector/events/ + - JSON files with current blocking status /var/lib/detector/status/ + - Internal data /var/lib/detector/_internal/ + +The tree under /var/lib/detector/ is served by Nginx with the exception of _internal + +Events are defined as changes between blocking and non-blocking on single +CC / test_name / input tuples + +Outputs are "upserted" where possible. New runs overwrite/update old data. + +Runs as a service "detector" in a systemd unit and sandbox + +See README.adoc +""" + +# Compatible with Python3.6 and 3.7 - linted with Black +# debdeps: python3-setuptools + +from argparse import ArgumentParser +from collections import namedtuple, deque +from configparser import ConfigParser +from datetime import date, datetime, timedelta +from pathlib import Path +from site import getsitepackages +import hashlib +import logging +import os +import pickle +import select +import signal +import sys + +from systemd.journal import JournalHandler # debdeps: python3-systemd +import psycopg2 # debdeps: python3-psycopg2 +import psycopg2.extensions +import psycopg2.extras +import ujson # debdeps: python3-ujson +import feedgenerator # debdeps: python3-feedgenerator + +from detector.metrics import setup_metrics +import detector.scoring as scoring + +log = logging.getLogger("detector") +metrics = setup_metrics(name="detector") + +DB_USER = "shovel" +DB_NAME = "metadb" +DB_PASSWORD = "yEqgNr2eXvgG255iEBxVeP" # This is already made public + +RO_DB_USER = "amsapi" +RO_DB_PASSWORD = "b2HUU6gKM19SvXzXJCzpUV" # This is already made public + +DEFAULT_STARTTIME = datetime(2016, 1, 1) + +BASEURL = "http://fastpath.ooni.nu:8080" +WEBAPP_URL = BASEURL + "/webapp" + +PKGDIR = getsitepackages()[-1] + +conf = None +cc_to_country_name = None # set by load_country_name_map + +# Speed up psycopg2's JSON load +psycopg2.extras.register_default_jsonb(loads=ujson.loads, globally=True) +psycopg2.extras.register_default_json(loads=ujson.loads, globally=True) + + +def fetch_past_data(conn, start_date): + """Fetch past data in large chunks ordered by measurement_start_time + """ + q = """ + SELECT + coalesce(false) as anomaly, + coalesce(false) as confirmed, + input, + measurement_start_time, + probe_cc, + scores::text, + coalesce('') as report_id, + test_name, + tid + FROM fastpath + WHERE measurement_start_time >= %(start_date)s + AND measurement_start_time < %(end_date)s + + UNION + + SELECT + anomaly, + confirmed, + input, + measurement_start_time, + probe_cc, + coalesce('') as scores, + report_id, + test_name, + coalesce('') as tid + + FROM measurement + JOIN report ON report.report_no = measurement.report_no + JOIN input ON input.input_no = measurement.input_no + WHERE measurement_start_time >= %(start_date)s + AND measurement_start_time < %(end_date)s + ORDER BY measurement_start_time + """ + assert start_date + + end_date = start_date + timedelta(weeks=1) + + chunk_size = 20000 + with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: + while True: + # Iterate across time blocks + now = datetime.utcnow() + # Ignore measurements with future timestamp + if end_date > now: + end_date = now + log.info("Last run") + log.info("Query from %s to %s", start_date, end_date) + p = dict(start_date=str(start_date), end_date=str(end_date)) + cur.execute(q, p) + while True: + # Iterate across chunks of rows + rows = cur.fetchmany(chunk_size) + if not rows: + break + log.info("Fetched msmt chunk of size %d", len(rows)) + for r in rows: + d = dict(r) + if d["scores"]: + d["scores"] = ujson.loads(d["scores"]) + + yield d + + if end_date == now: + break + + start_date += timedelta(weeks=1) + end_date += timedelta(weeks=1) + + +def fetch_past_data_selective(conn, start_date, cc, test_name, inp): + """Fetch past data in large chunks + """ + chunk_size = 200_000 + q = """ + SELECT + coalesce(false) as anomaly, + coalesce(false) as confirmed, + input, + measurement_start_time, + probe_cc, + probe_asn, + scores::text, + test_name, + tid + FROM fastpath + WHERE measurement_start_time >= %(start_date)s + AND probe_cc = %(cc)s + AND test_name = %(test_name)s + AND input = %(inp)s + + UNION + + SELECT + anomaly, + confirmed, + input, + measurement_start_time, + probe_cc, + probe_asn, + coalesce('') as scores, + test_name, + coalesce('') as tid + + FROM measurement + JOIN report ON report.report_no = measurement.report_no + JOIN input ON input.input_no = measurement.input_no + WHERE measurement_start_time >= %(start_date)s + AND probe_cc = %(cc)s + AND test_name = %(test_name)s + AND input = %(inp)s + + ORDER BY measurement_start_time + """ + p = dict(cc=cc, inp=inp, start_date=start_date, test_name=test_name) + + with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: + cur.execute(q, p) + while True: + rows = cur.fetchmany(chunk_size) + if not rows: + break + log.info("Fetched msmt chunk of size %d", len(rows)) + for r in rows: + d = dict(r) + if d["scores"]: + d["scores"] = ujson.loads(d["scores"]) + + yield d + + +def backfill_scores(d): + """Generate scores dict for measurements from the traditional pipeline + """ + if d.get("scores", None): + return + b = ( + scoring.anomaly + if d["anomaly"] + else 0 + scoring.confirmed + if d["confirmed"] + else 0 + ) + d["scores"] = dict(blocking_general=b) + + +def detect_blocking_changes_1s_g(g, cc, test_name, inp, start_date): + """Used by webapp + :returns: (msmts, changes) + """ + means = {} + msmts = [] + changes = [] + + for msm in g: + backfill_scores(msm) + k = (msm["probe_cc"], msm["test_name"], msm["input"]) + assert isinstance(msm["scores"], dict), repr(msm["scores"]) + change = detect_blocking_changes(means, msm, warmup=True) + date, mean, bblocked = means[k] + val = msm["scores"]["blocking_general"] + if change: + changes.append(change) + + msmts.append((date, val, mean)) + + log.debug("%d msmts processed", len(msmts)) + assert isinstance(msmts[0][0], datetime) + return (msmts, changes) + + +def detect_blocking_changes_one_stream(conn, cc, test_name, inp, start_date): + """Used by webapp + :returns: (msmts, changes) + """ + # TODO: move into webapp? + g = fetch_past_data_selective(conn, start_date, cc, test_name, inp) + return detect_blocking_changes_1s_g(g, cc, test_name, inp, start_date) + + +def load_asn_db(): + db_f = conf.vardir / "ASN.csv" + log.info("Loading %s", db_f) + if not db_f.is_file(): + log.info("No ASN file") + return {} + + d = {} + with db_f.open() as f: + for line in f: + try: + asn, name = line.split(",", 1) + asn = int(asn) + name = name.strip()[1:-2].strip() + d[asn] = name + except: + continue + + log.info("%d ASNs loaded", len(d)) + return d + + +def prevent_future_date(msm): + """If the msmt time is in the future replace it with utcnow + """ + # Timestamp are untrusted as they are generated by the probes + # This makes the process non-deterministic and non-reproducible + # but we can run unit tests against good inputs or mock utctime + # + # Warning: measurement_start_time is used for ranged queries against the DB + # and to pinpoint measurements and changes + now = datetime.utcnow() + if msm["measurement_start_time"] > now: + delta = msm["measurement_start_time"] - now + some_id = msm.get("tid", None) or msm.get("report_id", "") + log.info("Masking measurement %s %s in the future", some_id, delta) + msm["measurement_start_time"] = now + + +def detect_blocking_changes_asn_one_stream(conn, cc, test_name, inp, start_date): + """Used by webapp + :returns: (msmts, changes) + """ + g = fetch_past_data_selective(conn, start_date, cc, test_name, inp) + means = {} + msmts = [] + changes = [] + asn_breakdown = {} + + for msm in g: + backfill_scores(msm) + prevent_future_date(msm) + k = (msm["probe_cc"], msm["test_name"], msm["input"]) + assert isinstance(msm["scores"], dict), repr(msm["scores"]) + change = detect_blocking_changes(means, msm, warmup=True) + date, mean, _ = means[k] + val = msm["scores"]["blocking_general"] + if change: + changes.append(change) + + msmts.append((date, val, mean)) + del date + del val + del mean + del change + + # Generate charts for popular AS + asn = msm["probe_asn"] + a = asn_breakdown.get(asn, dict(means={}, msmts=[], changes=[])) + change = detect_blocking_changes(a["means"], msm, warmup=True) + date, mean, _ = a["means"][k] + val = msm["scores"]["blocking_general"] + a["msmts"].append((date, val, mean)) + if change: + a["changes"].append(change) + asn_breakdown[asn] = a + + log.debug("%d msmts processed", len(msmts)) + return (msmts, changes, asn_breakdown) + + +Change = namedtuple( + "Change", + [ + "probe_cc", + "test_name", + "input", + "blocked", + "mean", + "measurement_start_time", + "tid", + "report_id", + ], +) + +MeanStatus = namedtuple("MeanStatus", ["measurement_start_time", "val", "blocked"]) + + +def detect_blocking_changes(means: dict, msm: dict, warmup=False): + """Detect changes in blocking patterns + :returns: Change or None + """ + # TODO: move out params + upper_limit = 0.10 + lower_limit = 0.05 + # P: averaging value + # p=1: no averaging + # p=0.000001: very strong averaging + p = 0.02 + + inp = msm["input"] + if inp is None: + return + + if not isinstance(inp, str): + # Some inputs are lists. TODO: handle them? + log.debug("odd input") + return + + k = (msm["probe_cc"], msm["test_name"], inp) + tid = msm.get("tid", None) + report_id = msm.get("report_id", None) or None + + assert isinstance(msm["scores"], dict), repr(msm["scores"]) + blocking_general = msm["scores"]["blocking_general"] + measurement_start_time = msm["measurement_start_time"] + assert isinstance(measurement_start_time, datetime), repr(measurement_start_time) + + if k not in means: + # cc/test_name/input tuple never seen before + blocked = blocking_general > upper_limit + means[k] = MeanStatus(measurement_start_time, blocking_general, blocked) + if blocked: + if not warmup: + log.info("%r new and blocked", k) + metrics.incr("detected_blocked") + + return Change( + measurement_start_time=measurement_start_time, + blocked=blocked, + mean=blocking_general, + probe_cc=msm["probe_cc"], + input=msm["input"], + test_name=msm["test_name"], + tid=tid, + report_id=report_id, + ) + + else: + return None + + old = means[k] + assert isinstance(old, MeanStatus) + # tdelta = measurement_start_time - old.time + # TODO: average weighting by time delta; add timestamp to status and means + # TODO: record msm leading to status change + new_val = (1 - p) * old.val + p * blocking_general + means[k] = MeanStatus(measurement_start_time, new_val, old.blocked) + + if old.blocked and new_val < lower_limit: + # blocking cleared + means[k] = MeanStatus(measurement_start_time, new_val, False) + if not warmup: + log.info("%r cleared %.2f", k, new_val) + metrics.incr("detected_cleared") + + return Change( + measurement_start_time=measurement_start_time, + blocked=False, + mean=new_val, + probe_cc=msm["probe_cc"], + input=msm["input"], + test_name=msm["test_name"], + tid=tid, + report_id=report_id, + ) + + if not old.blocked and new_val > upper_limit: + means[k] = MeanStatus(measurement_start_time, new_val, True) + if not warmup: + log.info("%r blocked %.2f", k, new_val) + metrics.incr("detected_blocked") + + return Change( + measurement_start_time=measurement_start_time, + blocked=True, + mean=new_val, + probe_cc=msm["probe_cc"], + input=msm["input"], + test_name=msm["test_name"], + tid=tid, + report_id=report_id, + ) + + +def parse_date(d): + return datetime.strptime(d, "%Y-%m-%d").date() + + +def setup_dirs(conf, root): + """Setup directories, creating them if needed + """ + conf.vardir = root / "var/lib/detector" + conf.outdir = conf.vardir / "output" + conf.rssdir = conf.outdir / "rss" + conf.rssdir_by_cc = conf.rssdir / "by-country" + conf.rssdir_by_tname_inp = conf.rssdir / "type-inp" + conf.rssdir_by_cc_tname_inp = conf.rssdir / "cc-type-inp" + conf.eventdir = conf.outdir / "events" + conf.statusdir = conf.outdir / "status" + conf.pickledir = conf.outdir / "_internal" + for p in ( + conf.vardir, + conf.outdir, + conf.rssdir, + conf.rssdir_by_cc, + conf.rssdir_by_tname_inp, + conf.rssdir_by_cc_tname_inp, + conf.eventdir, + conf.statusdir, + conf.pickledir, + ): + p.mkdir(parents=True, exist_ok=True) + + +def setup(): + os.environ["TZ"] = "UTC" + global conf + ap = ArgumentParser(__doc__) + ap.add_argument("--devel", action="store_true", help="Devel mode") + ap.add_argument("--webapp", action="store_true", help="Run webapp") + ap.add_argument("--start-date", type=lambda d: parse_date(d)) + ap.add_argument("--db-host", default=None, help="Database hostname") + ap.add_argument( + "--ro-db-host", default=None, help="Read-only database hostname" + ) + conf = ap.parse_args() + if conf.devel: + format = "%(relativeCreated)d %(process)d %(levelname)s %(name)s %(message)s" + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=format) + else: + log.addHandler(JournalHandler(SYSLOG_IDENTIFIER="detector")) + log.setLevel(logging.DEBUG) + + + # Run inside current directory in devel mode + root = Path(os.getcwd()) if conf.devel else Path("/") + conf.conffile = root / "etc/detector.conf" + log.info("Using conf file %r", conf.conffile.as_posix()) + cp = ConfigParser() + with open(conf.conffile) as f: + cp.read_file(f) + conf.db_host = conf.db_host or cp["DEFAULT"]["db-host"] + conf.ro_db_host = conf.ro_db_host or cp["DEFAULT"]["ro-db-host"] + + setup_dirs(conf, root) + + +@metrics.timer("handle_new_measurement") +def handle_new_msg(msg, means, rw_conn): + """Handle one measurement received in realtime from PostgreSQL + notifications + """ + msm = ujson.loads(msg.payload) + assert isinstance(msm["scores"], dict), type(msm["scores"]) + msm["measurement_start_time"] = datetime.strptime( + msm["measurement_start_time"], "%Y-%m-%d %H:%M:%S" + ) + log.debug("Notify for msmt from %s", msm.get("probe_cc", "")) + prevent_future_date(msm) + change = detect_blocking_changes(means, msm, warmup=False) + if change is not None: + upsert_change(change) + + +def connect_to_db(db_host, db_user, db_name, db_password): + dsn = f"host={db_host} user={db_user} dbname={db_name} password={db_password}" + log.info("Connecting to database: %r", dsn) + conn = psycopg2.connect(dsn) + conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + return conn + + +def snapshot_means(msm, last_snapshot_date, means): + """Save means to disk every month + """ + # TODO: add config parameter to recompute past data + t = msm["measurement_start_time"] + month = date(t.year, t.month, 1) + if month == last_snapshot_date: + return last_snapshot_date + + log.info("Saving %s monthly snapshot", month) + save_means(means, month) + return month + + +def process_historical_data(ro_conn, rw_conn, start_date, means): + """Process past data + """ + assert start_date + log.info("Running process_historical_data from %s", start_date) + t = metrics.timer("process_historical_data").start() + cnt = 0 + # fetch_past_data returns measurements ordered by measurement_start_time + last_snap = None + for past_msm in fetch_past_data(ro_conn, start_date): + backfill_scores(past_msm) + prevent_future_date(past_msm) + last_snap = snapshot_means(past_msm, last_snap, means) + change = detect_blocking_changes(means, past_msm, warmup=True) + cnt += 1 + if change is not None: + upsert_change(change) + + metrics.incr("processed_msmt") + + t.stop() + for m in means.values(): + assert isinstance(m[2], bool), m + + blk_cnt = sum(m[2] for m in means.values()) # count blocked + p = 100 * blk_cnt / len(means) + log.info("%d tracked items, %d blocked (%.3f%%)", len(means), blk_cnt, p) + log.info("Processed %d measurements. Speed: %d K-items per second", cnt, cnt / t.ms) + + +def create_url(change): + return f"{WEBAPP_URL}/chart?cc={change.probe_cc}&test_name={change.test_name}&input={change.input}&start_date=" + + +def basefn(cc, test_name, inp): + """Generate opaque filesystem-safe filename + inp can be "" or None (returning different hashes) + """ + d = f"{cc}:{test_name}:{inp}" + h = hashlib.shake_128(d.encode()).hexdigest(16) + return h + + +# TODO rename changes to events? + + +def explorer_url(c: Change) -> str: + return f"https://explorer.ooni.org/measurement/{c.report_id}?input={c.input}" + + +# TODO: regenerate RSS feeds (only) once after the warmup terminates + + +@metrics.timer("write_feed") +def write_feed(feed, p: Path) -> None: + """Write out RSS feed atomically""" + tmp_p = p.with_suffix(".tmp") + with tmp_p.open("w") as f: + feed.write(f, "utf-8") + + tmp_p.rename(p) + + +global_feed_cache = deque(maxlen=1000) + + +@metrics.timer("update_rss_feed_global") +def update_rss_feed_global(change: Change) -> None: + """Generate RSS feed for global events and write it in + /var/lib/detector/rss/global.xml + """ + # The files are served by Nginx + global global_feed_cache + if not change.input: + return + global_feed_cache.append(change) + feed = feedgenerator.Rss201rev2Feed( + title="OONI events", + link="https://explorer.ooni.org", + description="Blocked services and websites detected by OONI", + language="en", + ) + for c in global_feed_cache: + un = "" if c.blocked else "un" + country = cc_to_country_name.get(c.probe_cc.upper(), c.probe_cc) + feed.add_item( + title=f"{c.input} {un}blocked in {country}", + link=explorer_url(c), + description=f"Change detected on {c.measurement_start_time}", + pubdate=c.measurement_start_time, + updateddate=datetime.utcnow(), + ) + write_feed(feed, conf.rssdir / "global.xml") + + +by_cc_feed_cache = {} + + +@metrics.timer("update_rss_feed_by_country") +def update_rss_feed_by_country(change: Change) -> None: + """Generate RSS feed for events grouped by country and write it in + /var/lib/detector/rss/by-country/.xml + """ + # The files are served by Nginx + global by_cc_feed_cache + if not change.input: + return + cc = change.probe_cc + if cc not in by_cc_feed_cache: + by_cc_feed_cache[cc] = deque(maxlen=1000) + by_cc_feed_cache[cc].append(change) + feed = feedgenerator.Rss201rev2Feed( + title=f"OONI events in {cc}", + link="https://explorer.ooni.org", + description="Blocked services and websites detected by OONI", + language="en", + ) + for c in by_cc_feed_cache[cc]: + un = "" if c.blocked else "un" + country = cc_to_country_name.get(c.probe_cc.upper(), c.probe_cc) + feed.add_item( + title=f"{c.input} {un}blocked in {country}", + link=explorer_url(c), + description=f"Change detected on {c.measurement_start_time}", + pubdate=c.measurement_start_time, + updateddate=datetime.utcnow(), + ) + write_feed(feed, conf.rssdir_by_cc / f"{cc}.xml") + + +@metrics.timer("update_rss_feeds_by_cc_tname_inp") +def update_rss_feeds_by_cc_tname_inp(events, hashfname): + """Generate RSS feed by cc / test_name / input and write + /var/lib/detector/rss/cc-type-inp/.xml + """ + # The files are served by Nginx + feed = feedgenerator.Rss201rev2Feed( + title="OONI events", + link="https://explorer.ooni.org", + description="Blocked services and websites detected by OONI", + language="en", + ) + # TODO: render date properly and add blocked/unblocked + # TODO: put only recent events in the feed (based on the latest event time) + for e in events: + if not e["input"]: + continue + + country = cc_to_country_name.get(c.probe_cc.upper(), c.probe_cc) + feed.add_item( + title=f"{c.input} {un}blocked in {country}", + link=explorer_url(c), + description=f"Change detected on {c.measurement_start_time}", + pubdate=c.measurement_start_time, + updateddate=datetime.utcnow(), + ) + + write_feed(feed, conf.rssdir / f"{hashfname}.xml") + + +def update_status_files(blocking_events): + # The files are served by Nginx + return # FIXME + + # This contains the last status change for every cc/test_name/input + # that ever had a block/unblock event + status = {k: v[-1] for k, v in blocking_events.items()} + + statusfile = conf.statusdir / f"status.json" + d = dict(format=1, status=status) + with statusfile.open("w") as f: + ujson.dump(d, f) + + log.debug("Wrote %s", statusfile) + + +@metrics.timer("upsert_change") +def upsert_change(change): + """Create / update RSS and JSON files with a new change + """ + # Create DB tables in future if needed + debug_url = create_url(change) + log.info("Change! %r %r", change, debug_url) + if not change.report_id: + log.error("Empty report_id") + return + + try: + update_rss_feed_global(change) + update_rss_feed_by_country(change) + except Exception as e: + log.error(e, exc_info=1) + + # TODO: currently unused + return + + # Append change to a list in a JSON file + # It contains all the block/unblock events for a given cc/test_name/input + hashfname = basefn(change.probe_cc, change.test_name, change.input) + events_f = conf.eventdir / f"{hashfname}.json" + if events_f.is_file(): + with events_f.open() as f: + ecache = ujson.load(f) + else: + ecache = dict(format=1, blocking_events=[]) + + ecache["blocking_events"].append(change._asdict()) + log.info("Saving %s", events_f) + with events_f.open("w") as f: + ujson.dump(ecache, f) + + update_rss_feeds_by_cc_tname_inp(ecache["blocking_events"], hashfname) + + update_status_files(ecache["blocking_events"]) + + +def load_means(): + """Load means from a pkl file + The file is safely owned by the detector. + """ + pf = conf.pickledir / "means.pkl" + log.info("Loading means from %s", pf) + if pf.is_file(): + perms = pf.stat().st_mode + assert (perms & 2) == 0, "Insecure pickle permissions %s" % oct(perms) + with pf.open("rb") as f: + means = pickle.load(f) + + assert means + earliest = min(m.measurement_start_time for m in means.values()) + latest = max(m.measurement_start_time for m in means.values()) + # Cleanup + # t = datetime.utcnow() + # for k, m in means.items(): + # if m.measurement_start_time > t: + # log.info("Fixing time") + # means[k] = MeanStatus(t, m.val, m.blocked) + + latest = max(m[0] for m in means.values()) + log.info("Earliest mean: %s", earliest) + return means, latest + + log.info("Creating new means file") + return {}, None + + +def save_means(means, date): + """Save means atomically. Protocol 4 + """ + tstamp = date.strftime(".%Y-%m-%d") if date else "" + pf = conf.pickledir / f"means{tstamp}.pkl" + pft = pf.with_suffix(".tmp") + if not means: + log.error("No means to save") + return + log.info("Saving %d means to %s", len(means), pf) + latest = max(m[0] for m in means.values()) + log.info("Latest mean: %s", latest) + with pft.open("wb") as f: + pickle.dump(means, f, protocol=4) + pft.rename(pf) + log.info("Saving completed") + + +# FIXME: for performance reasons we want to minimize heavy DB queries. +# Means are cached in a pickle file to allow restarts and we pick up where +# we stopped based on measurement_start_time. Yet the timestamp are untrusted +# as they are generated by the probes. + + +def load_country_name_map(): + """Load country-list.json and create a lookup dictionary + """ + fi = f"{PKGDIR}/detector/data/country-list.json" + log.info("Loading %s", fi) + with open(fi) as f: + clist = ujson.load(f) + + # The file is deployed with the detector: crash out if it's broken + d = {} + for c in clist: + cc = c["iso3166_alpha2"] + name = c["name"] + assert cc and (len(cc) == 2) and name + d[cc.upper()] = name + + log.info("Loaded %d country names", len(d)) + return d + + +# TODO: handle input = None both in terms of filename collision and RSS feed +# and add functional tests + + +def main(): + setup() + log.info("Starting") + + global cc_to_country_name + cc_to_country_name = load_country_name_map() + + ro_conn = connect_to_db(conf.ro_db_host, RO_DB_USER, DB_NAME, RO_DB_PASSWORD) + + if conf.webapp: + import detector.detector_webapp as wa + + wa.db_conn = ro_conn + wa.asn_db = load_asn_db() + log.info("Starting webapp") + wa.bottle.TEMPLATE_PATH.insert(0, f"{PKGDIR}/detector/views") + wa.bottle.run(port=8880, debug=conf.devel) + log.info("Exiting webapp") + return + + means, latest_mean = load_means() + log.info("Latest mean: %s", latest_mean) + + # Register exit handlers + def save_means_on_exit(*a): + log.info("Received SIGINT / SIGTERM") + save_means(means, None) + log.info("Exiting") + sys.exit() + + signal.signal(signal.SIGINT, save_means_on_exit) + signal.signal(signal.SIGTERM, save_means_on_exit) + + rw_conn = connect_to_db(conf.db_host, DB_USER, DB_NAME, DB_PASSWORD) + + td = timedelta(weeks=6) + start_date = latest_mean - td if latest_mean else DEFAULT_STARTTIME + process_historical_data(ro_conn, rw_conn, start_date, means) + save_means(means, None) + + log.info("Starting real-time processing") + with rw_conn.cursor() as cur: + cur.execute("LISTEN fastpath;") + + while True: + if select.select([rw_conn], [], [], 60) == ([], [], []): + continue # timeout + + rw_conn.poll() + while rw_conn.notifies: + msg = rw_conn.notifies.pop(0) + try: + handle_new_msg(msg, means, rw_conn) + except Exception as e: + log.exception(e) + + +if __name__ == "__main__": + main() diff --git a/detector/detector/detector_webapp.py b/detector/detector/detector_webapp.py new file mode 100644 index 00000000..23ab5f78 --- /dev/null +++ b/detector/detector/detector_webapp.py @@ -0,0 +1,170 @@ +""" +Detector web application + +Currently used only for tuning the detector, it runs event detection for one +cc / test_name / input independently from the event detector daemon. +The output are only displayed in charts and not used to generate RSS feeds +or other. + +""" + +# TODO: cleanup + +from datetime import datetime, timedelta +import logging +import json + +from bottle import request +import bottle + +from detector.detector import ( + detect_blocking_changes_asn_one_stream, +) + +from detector.metrics import setup_metrics + +log = logging.getLogger("detector") +metrics = setup_metrics(name="detector") + +db_conn = None # Set by detector.py or during functional testing + +asn_db = None # Set by detector.py + +def _datetime_handler(x): + if isinstance(x, datetime): + return x.isoformat() + raise TypeError("unknown type") + + +bottle.install( + bottle.JSONPlugin(json_dumps=lambda o: json.dumps(o, default=_datetime_handler)) +) + + +def generate_chart(start_d, end_d, msmts, changes, title): + """Render measurements and changes into a SVG chart + :returns: dict + """ + assert isinstance(msmts[0][0], datetime) + x1 = 100 + x2 = 1100 + y1 = 50 + y2 = 300 + # scale x + delta = (end_d - start_d).total_seconds() + assert delta != 0 + x_scale = (x2 - x1) / delta + + return dict( + msmts=msmts, + changes=changes, + x_scale=x_scale, + start_d=start_d, + end_d=end_d, + x1=x1, + x2=x2, + y1=y1, + y2=y2, + title=title, + ) + + +@bottle.route("/") +@bottle.view("form") +def index(): + log.debug("Serving index") + return {} + + +def plot_series(conn, ccs, test_names, inputs, start_date, split_asn): + """Generates time-series for CC / test_name / input + to be rendered as SVG charts + :returns: list of charts + """ + log.error(repr(split_asn)) + charts = [] + for cc in ccs: + for test_name in test_names: + for inp in inputs: + log.info("Generating chart for %r %r %r", cc, test_name, inp) + # TODO: merge inputs here and in event detection? + + (msmts, changes, asn_breakdown) = detect_blocking_changes_asn_one_stream( + conn, cc, test_name, inp, start_date + ) + if len(msmts) < 2: + log.debug("Not enough data") + continue + + # Time range + assert isinstance(msmts[0][0], datetime) + start_d = min(e[0] for e in msmts) + end_d = max(e[0] for e in msmts) + delta = (end_d - start_d).total_seconds() + assert delta > 0 + log.debug(delta) + + title = f"{cc} {test_name} {inp} {start_d} - {end_d}" + country_chart = generate_chart(start_d, end_d, msmts, changes, title) + + charts.append(country_chart) + + if split_asn: + # Most popular ASNs + popular = sorted( + asn_breakdown, key=lambda asn: len(asn_breakdown[asn]["msmts"]), reverse=True + ) + popular = popular[:20] + for asn in popular: + title = "AS{} {}".format(asn, asn_db.get(asn, "")) + a = asn_breakdown[asn] + try: + c = generate_chart(start_d, end_d, a["msmts"], a["changes"], title) + charts.append(c) + except: + log.error(a) + + return charts + + + +@bottle.route("/chart") +@bottle.view("page") +@metrics.timer("generate_charts") +def genchart(): + params = ("ccs", "test_names", "inputs", "start_date", "split_asn") + q = {k: (request.query.get(k, "").strip() or None) for k in params} + assert q["ccs"], "missing ccs query param" + + ccs = [i.strip() for i in q["ccs"].split(",") if i.strip()] + for cc in ccs: + assert len(cc) == 2, "CC must be 2 letters" + + test_names = q["test_names"].split(",") or ["web_connectivity",] + inputs = q["inputs"] + assert inputs, "Inputs are required" + inputs = [i.strip() for i in inputs.split(",") if i.strip()] + split_asn = q["split_asn"] is not None + start_date = q["start_date"] + if start_date: + start_date = datetime.strptime(start_date, "%Y-%m-%d") + else: + start_date = datetime.now() - timedelta(days=10) + + log.debug("Serving query %s %s %s %s", ccs, test_names, inputs, start_date) + + charts = plot_series(db_conn, ccs, test_names, inputs, start_date, split_asn) + form = dict( + inputs=",".join(inputs), + test_names=",".join(test_names), + ccs=",".join(ccs), + start_date=start_date.strftime("%Y-%m-%d"), + split_asn=split_asn, + ) + return dict(charts=charts, title="Detector", form=form) + + +@bottle.error(500) +def error_handler_500(error): + log.error(error.exception) + return repr(error.exception) diff --git a/detector/detector/metrics.py b/detector/detector/metrics.py new file mode 120000 index 00000000..55394470 --- /dev/null +++ b/detector/detector/metrics.py @@ -0,0 +1 @@ +../../fastpath/fastpath/metrics.py \ No newline at end of file diff --git a/detector/detector/scoring.py b/detector/detector/scoring.py new file mode 100644 index 00000000..7b20d222 --- /dev/null +++ b/detector/detector/scoring.py @@ -0,0 +1,9 @@ +""" +OONI Fastpath + +Scoring parameters +""" + + +anomaly = 0.8 +confirmed = 2.0 diff --git a/detector/detector/tests/test_unit.py b/detector/detector/tests/test_unit.py new file mode 100644 index 00000000..9c8b3f0c --- /dev/null +++ b/detector/detector/tests/test_unit.py @@ -0,0 +1,221 @@ +# +# Fastpath - event detector unit tests +# + +from datetime import datetime +from pathlib import Path +import json + + +import bottle +import pytest + +import detector.detector as dt +import detector.detector_webapp as webapp + +bottle.TEMPLATE_PATH.insert(0, "detector/views") +data = Path("detector/tests/data") + + +def datadir(p): + return data / p + + +def save(o, p): + data.joinpath(p).write_text(json.dumps(o, sort_keys=True, default=lambda x: str(x))) + + +def load(p): + with data.joinpath(p).open() as f: + return json.load(f) + + +def jd(o): + return json.dumps(o, indent=2, sort_keys=True, default=lambda x: str(x)) + + +def trim(chart): + [chart.pop(k) for k in list(chart) if k not in ("msmts", "changes")] + + +@pytest.fixture(scope="session") +def dbconn(): + # Used manually to access the database to gather and save test data + # when writing new tests. Not used during unit testing. + conn = dt.connect_to_db("127.0.0.1", "readonly", dt.DB_NAME, "") + webapp.db_conn = conn + return conn + + +def notest_chart_ww_BR_1(dbconn): + cc = "BR" + inp = "https://www.womenonwaves.org/" + start_date = datetime(2018, 11, 1) + test_name = "web_connectivity" + chart = webapp.plot_series(webapp.db_conn, cc, test_name, inp, start_date) + [chart.pop(k) for k in chart if k not in ("events", "changes")] + expected = datadir("detector_chart_1.svg").read_text() + assert chart == expected + + +def dbgenerator(fname): + ## mock DB query + for row in load(fname): + st = row["measurement_start_time"] + if isinstance(st, int): + row["measurement_start_time"] = datetime.utcfromtimestamp(st) + else: + row["measurement_start_time"] = datetime.strptime(st, "%Y-%m-%d %H:%M:%S") + + yield row + + +def urgh(o): + # FIXME: temporary hack to transform datetimes and truncate float numbers + # to allow comparison + return json.loads(jd(o)) + + +def notest_chart_ww_BR_2(): + cc = "BR" + inp = "https://www.womenonwaves.org/" + start_date = datetime(2019, 1, 1) + test_name = "web_connectivity" + + g = dbgenerator("detector_query_ww_BR_2.json") + + (msmts, changes) = dt.detect_blocking_changes_1s_g( + g, cc, test_name, inp, start_date + ) + # save(dict(msmts=msmts, changes=changes) , "detector_ww_BR_2_output.json") + + expected = load("detector_ww_BR_2_output.json") + assert len(msmts) == 809 + assert urgh(msmts)[0] == expected["msmts"][0] + assert urgh(msmts) == expected["msmts"] + assert urgh(changes) == expected["changes"] + + +def test_chart_ww_BR_2019_generate_chart(): + cc = "BR" + inp = "https://www.womenonwaves.org/" + start_date = datetime(2019, 1, 1) + test_name = "web_connectivity" + + g = dbgenerator("detector_query_ww_BR_2.json") + + (msmts, changes) = dt.detect_blocking_changes_1s_g( + g, cc, test_name, inp, start_date + ) + assert isinstance(msmts[0][0], datetime) + # save(dict(msmts=msmts, changes=changes) , "detector_ww_BR_2019_output.json") + expected = load("detector_ww_BR_2019_output.json") + assert len(msmts) == 809 + u = urgh(msmts) + for i in range(len(msmts)): + assert u[i] == expected["msmts"][i] + + assert len(changes) == len(expected["changes"]) + for e, c in zip(expected["changes"], changes): + ts, m, oldcode = e + assert ts == str(c.measurement_start_time) + assert m == c.mean + + with Path("detector/views/chart_alone.tpl").open() as f: + tpl = webapp.bottle.SimpleTemplate(f.read()) + + cd = webapp.generate_chart(msmts, changes, cc, test_name, inp) + chart = tpl.render(**cd) + assert chart + data.joinpath("output/chart_ww_BR_2019.html").write_text(chart) + + +def test_chart_ww_BR_2018_generate_chart(): + cc = "BR" + inp = "https://www.womenonwaves.org/" + start_date = datetime(2018, 11, 1) + test_name = "web_connectivity" + + # g = dt.fetch_past_data_selective(dbconn, start_date, cc, test_name, inp) + # save(list(g), "detector_query_ww_BR_2018.json") + g = dbgenerator("detector_query_ww_BR_2018.json") + + (msmts, changes) = dt.detect_blocking_changes_1s_g( + g, cc, test_name, inp, start_date + ) + assert isinstance(msmts[0][0], datetime) + # save(dict(msmts=msmts, changes=changes) , "detector_ww_BR_2018_output.json") + expected = load("detector_ww_BR_2018_output.json") + assert len(msmts) == 911 + u = urgh(msmts) + for i in range(len(msmts)): + assert u[i] == expected["msmts"][i] + + assert len(changes) == len(expected["changes"]) + for e, c in zip(expected["changes"], changes): + ts, m, oldcode = e + assert ts == str(c.measurement_start_time) + assert m == c.mean + + with Path("detector/views/chart_alone.tpl").open() as f: + tpl = webapp.bottle.SimpleTemplate(f.read()) + + cd = webapp.generate_chart(msmts, changes, cc, test_name, inp) + chart = tpl.render(**cd) + assert chart + data.joinpath("output/chart_ww_BR_2018.html").write_text(chart) + + +def bench_detect_blocking_changes_1s_g(g): + cc = "BR" + inp = "foo" + start_date = datetime(2019, 1, 1) + test_name = "web_connectivity" + return dt.detect_blocking_changes_1s_g( + g, cc, test_name, inp, start_date + ) + + +def test_bench_detect_blocking_changes(benchmark): + # debdeps: python3-pytest-benchmark + g = [] + for x in range(1020): + v = { + "anomaly": None if (int(x / 100) % 2 == 0) else True, + "confirmed": None, + "input": "foo", + "measurement_start_time": datetime.utcfromtimestamp(x + 1234567890), + "probe_cc": "BR", + "scores": "", + "test_name": "web_connectivity", + "tid": "", + } + g.append(v) + (msmts, changes) = benchmark(bench_detect_blocking_changes_1s_g, g) + + with Path("detector/views/chart_alone.tpl").open() as f: + tpl = webapp.bottle.SimpleTemplate(f.read()) + + cc = "BR" + inp = "foo" + test_name = "web_connectivity" + cd = webapp.generate_chart(msmts, changes, cc, test_name, inp) + chart = tpl.render(**cd) + assert chart + data.joinpath("output/chart_ww_BR_bench.html").write_text(chart) + last_mean = msmts[-1][-1] + assert pytest.approx(0.415287511652) == last_mean + + +def notest_chart_ww_BR(): + cc = "BR" + inp = "https://www.womenonwaves.org/" + start_date = datetime(2018, 11, 1) + test_name = "web_connectivity" + chart1 = webapp.plot_series(webapp.db_conn, cc, test_name, inp, start_date) + start_date = datetime(2019, 1, 1) + chart2 = webapp.plot_series(webapp.db_conn, cc, test_name, inp, start_date) + chart = chart1 + chart2 + # data.joinpath("detector_chart_ww_BR.svg").write_text(chart) + # expected = data.joinpath("detector_chart_2.svg").read_text() + # assert chart == expected diff --git a/detector/detector/views/chart.tpl b/detector/detector/views/chart.tpl new file mode 100644 index 00000000..33ac494f --- /dev/null +++ b/detector/detector/views/chart.tpl @@ -0,0 +1,60 @@ + + + {{title}} + + % pcx = pcy = None + % for d, val, mean in msmts: + % cx = (d - start_d).total_seconds() * x_scale + x1 + % cy = y2 - min(max(val, 0) * 200, 300) + % #r = "{:02x}".format(min(int(max(val, 0) * 170), 255)) + % col = "d60000" if val > .5 else "00d600" + + + % # moving average + % cy = y2 - min(max(mean, 0) * 200, 300) + % if pcy is not None: + + % end + % pcx, pcy = cx, cy + % end + + % # changes in blocking + % for c in changes: + % cx = (c.measurement_start_time - start_d).total_seconds() * x_scale + x1 + % col = "ff3333" if c.blocked else "33ff33" + + % end + + + % # start/end date labels + {{start_d}} + {{end_d}} + + + + + % for val in (0.0, 1.0, 2.0): + % cy = y2 - min(max(val, 0) * 100, (y2 - y1)) + + % end + diff --git a/detector/detector/views/chart_alone.tpl b/detector/detector/views/chart_alone.tpl new file mode 100644 index 00000000..aed6bcad --- /dev/null +++ b/detector/detector/views/chart_alone.tpl @@ -0,0 +1,58 @@ + + + {{cc}} {{test_name}} {{inp}} + + % pcx = pcy = None + % for d, val, mean in msmts: + % cx = (d - start_d).total_seconds() * x_scale + x1 + % cy = y2 - min(max(val, 0) * 100, 300) + % r = "{:02x}".format(min(int(max(val, 0) * 170), 255)) + + + % # moving average + % cy = y2 - min(max(mean, 0) * 100, 300) + % if pcy is not None: + + % end + % pcx, pcy = cx, cy + % end + + % # changes in blocking + % for c in changes: + % cx = (c.measurement_start_time - start_d).total_seconds() * x_scale + x1 + % col = "ff3333" if c.blocked else "33ff33" + + % end + + + {{start_d}} + {{d}} + + + + + % for val in (0.0, 1.0, 2.0): + % cy = y2 - min(max(val, 0) * 100, (y2 - y1)) + + % end + diff --git a/detector/detector/views/form.tpl b/detector/detector/views/form.tpl new file mode 100644 index 00000000..0e613d16 --- /dev/null +++ b/detector/detector/views/form.tpl @@ -0,0 +1,27 @@ + + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+ diff --git a/detector/detector/views/page.tpl b/detector/detector/views/page.tpl new file mode 100644 index 00000000..ac32ea4b --- /dev/null +++ b/detector/detector/views/page.tpl @@ -0,0 +1,136 @@ + + + + {{title}} + + + +
+ + + + + + + + + + + + + + + +
+ % for c in charts: + % msmts = c["msmts"] + % changes = c["changes"] + % x_scale = c["x_scale"] + % start_d = c["start_d"] + % end_d = c["end_d"] + % x1 = c["x1"] + % x2 = c["x2"] + % y1 = c["y1"] + % y2 = c["y2"] + % title = c["title"] + + + {{title}} + + % pcx = pcy = None + % for d, val, mean in msmts: + % cx = (d - start_d).total_seconds() * x_scale + x1 + % cy = y2 - min(max(val, 0) * 200, 300) + % #r = "{:02x}".format(min(int(max(val, 0) * 170), 255)) + % col = "d60000" if val > .5 else "00d600" + + + % # moving average + % cy = y2 - min(max(mean, 0) * 200, 300) + % if pcy is not None: + + % end + % pcx, pcy = cx, cy + % end + + % # changes in blocking + % for c in changes: + % cx = (c.measurement_start_time - start_d).total_seconds() * x_scale + x1 + % col = "ff3333" if c.blocked else "33ff33" + + % end + + + % # start/end date labels + {{start_d}} + {{end_d}} + + + + + % for val in (0.0, 1.0, 2.0): + % cy = y2 - min(max(val, 0) * 100, (y2 - y1)) + + % end + + % end + + diff --git a/detector/setup.py b/detector/setup.py new file mode 100644 index 00000000..3572e5bf --- /dev/null +++ b/detector/setup.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from setuptools import setup + +NAME = "detector" +DESCRIPTION = "" + +REQUIRED = [] + +setup( + name=NAME, + python_requires=">=3.6.0", + packages=["detector", "detector.tests"], + entry_points={"console_scripts": ["detector=detector.detector:main",]}, + install_requires=REQUIRED, + include_package_data=True, + zip_safe=False, + package_data={"detector": ["views/*.tpl", "static/*", "data/country-list.json"]}, +)