diff --git a/cicd/jenkins-slave-zap/centos/Dockerfile b/cicd/jenkins-slave-zap/centos/Dockerfile index 15b3b02..5cdf1d9 100644 --- a/cicd/jenkins-slave-zap/centos/Dockerfile +++ b/cicd/jenkins-slave-zap/centos/Dockerfile @@ -7,17 +7,18 @@ RUN yum install -y epel-release && \ RUN yum install -y redhat-rpm-config \ make automake autoconf gcc gcc-c++ \ libstdc++ libstdc++-devel \ - java-1.8.0-openjdk wget curl \ - xmlstarlet git x11vnc gettext tar \ - xorg-x11-server-Xvfb openbox xterm \ - net-tools python-pip \ - firefox nss_wrapper java-1.8.0-openjdk-headless \ - java-1.8.0-openjdk-devel nss_wrapper git && \ + wget curl git firefox \ + xmlstarlet gettext tar \ + x11vnc xorg-x11-server-Xvfb xterm \ + openbox net-tools nss_wrapper \ + python python-pip \ + java-11-openjdk-headless java-11-openjdk java-11-openjdk-devel && \ yum clean all + +# upgrade pip and install the latest dev version of the python API RUN pip install --upgrade pip RUN pip install zapcli -# Install latest dev version of the python API RUN pip install python-owasp-zap-v2.4 RUN mkdir -p /zap/wrk @@ -29,23 +30,26 @@ RUN mkdir -p /var/lib/jenkins/.vnc COPY configuration/* /var/lib/jenkins/ COPY configuration/run-jnlp-client /usr/local/bin/run-jnlp-client -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64/ ENV PATH $JAVA_HOME/bin:/zap:$PATH ENV ZAP_PATH /zap/zap.sh ENV HOME /var/lib/jenkins # Default port for use with zapcli -ENV ZAP_PORT 8080 +ENV ZAP_PORT=8080 COPY policies /var/lib/jenkins/.ZAP/policies/ COPY .xinitrc /var/lib/jenkins/ WORKDIR /zap + +ENV WEBSWING_VERSION=2.7.1 + # Download and expand the latest stable release RUN curl -s https://raw.githubusercontent.com/zaproxy/zap-admin/master/ZapVersions-dev.xml | xmlstarlet sel -t -v //url |grep -i Linux | wget -q --content-disposition -i - -O - | tar zx --strip-components=1 && \ - curl -s -L https://bitbucket.org/meszarv/webswing/downloads/webswing-2.3-distribution.zip | jar -x && \ + curl -s -L https://bitbucket.org/meszarv/webswing/downloads/webswing-${WEBSWING_VERSION}.zip | jar -x && \ touch AcceptedLicense -ADD webswing.config /zap/webswing-2.3/webswing.config +ADD webswing.config /zap/webswing-${WEBSWING_VERSION}/webswing.config RUN chown -R root:root /zap && \ chown -R root:root /var/lib/jenkins && \ diff --git a/cicd/jenkins-slave-zap/centos/zap/zap-api-scan.py b/cicd/jenkins-slave-zap/centos/zap/zap-api-scan.py index b958910..4066e64 100755 --- a/cicd/jenkins-slave-zap/centos/zap/zap-api-scan.py +++ b/cicd/jenkins-slave-zap/centos/zap/zap-api-scan.py @@ -55,19 +55,22 @@ import subprocess import sys import time -import urllib2 from datetime import datetime +from six.moves.urllib.parse import urljoin from zapv2 import ZAPv2 from zap_common import * -timeout = 120 + +class NoUrlsException(Exception): + pass + + config_dict = {} config_msg = {} out_of_scope_dict = {} -levels = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] min_level = 0 -# Scan rules that aren't really relevant, eg the examples rules in the alpha set +# Scan rules that aren't really relevant, e.g. the examples rules in the alpha set blacklist = ['-1', '50003', '60000', '60001'] # Scan rules that are being addressed @@ -80,15 +83,17 @@ def usage(): print('Usage: zap-api-scan.py -t -f [options]') - print(' -t target target API definition, OpenAPI or SOAP, local file or URL, eg https://www.example.com/openapi.json') + print(' -t target target API definition, OpenAPI or SOAP, local file or URL, e.g. https://www.example.com/openapi.json') print(' -f format either openapi or soap') print('Options:') + print(' -h print this help message') print(' -c config_file config file to use to INFO, IGNORE or FAIL warnings') print(' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') print(' -g gen_file generate default config file(all rules set to WARN)') print(' -r report_html file to write the full ZAP HTML report') print(' -w report_md file to write the full ZAP Wiki(Markdown) report') print(' -x report_xml file to write the full ZAP XML report') + print(' -J report_json file to write the full ZAP JSON document') print(' -a include the alpha passive scan rules as well') print(' -d show debug messages') print(' -P specify listen port') @@ -98,7 +103,11 @@ def usage(): print(' -n context_file context file which will be loaded prior to scanning the target') print(' -p progress_file progress file which specifies issues that are being addressed') print(' -s short output format - dont show PASSes or example URLs') + print(' -S safe mode this will skip the active scan and perform a baseline scan') + print(' -T max time in minutes to wait for ZAP to start and the passive scan to run') + print(' -O the hostname to override in the (remote) OpenAPI spec') print(' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') + print(' --hook path to python file that define your custom hooks') print('') print('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-API-Scan') @@ -118,16 +127,21 @@ def main(argv): report_html = '' report_md = '' report_xml = '' + report_json = '' target = '' target_file = '' target_url = '' + host_override = '' format = '' zap_alpha = False + baseline = False info_unspecified = False base_dir = '' zap_ip = 'localhost' zap_options = '' delay = 0 + timeout = 0 + hook_file = None pass_count = 0 warn_count = 0 @@ -138,14 +152,17 @@ def main(argv): fail_inprog_count = 0 try: - opts, args = getopt.getopt(argv, "t:f:c:u:g:m:n:r:w:x:l:daijp:sz:P:D:") + opts, args = getopt.getopt(argv, "t:f:c:u:g:m:n:r:J:w:x:l:hdaijSp:sz:P:D:T:O:", ["hook="]) except getopt.GetoptError as exc: logging.warning('Invalid option ' + exc.opt + ' : ' + exc.msg) usage() sys.exit(3) for opt, arg in opts: - if opt == '-t': + if opt == '-h': + usage() + sys.exit(0) + elif opt == '-t': target = arg logging.debug('Target: ' + target) elif opt == '-f': @@ -168,6 +185,8 @@ def main(argv): progress_file = arg elif opt == '-r': report_html = arg + elif opt == '-J': + report_json = arg elif opt == '-w': report_md = arg elif opt == '-x': @@ -178,15 +197,28 @@ def main(argv): info_unspecified = True elif opt == '-l': try: - min_level = levels.index(arg) + min_level = zap_conf_lvls.index(arg) except ValueError: - logging.warning('Level must be one of ' + str(levels)) + logging.warning('Level must be one of ' + str(zap_conf_lvls)) usage() sys.exit(3) elif opt == '-z': zap_options = arg elif opt == '-s': detailed_output = False + elif opt == '-S': + baseline = True + elif opt == '-T': + timeout = int(arg) + elif opt == '-O': + host_override = arg + elif opt == '--hook': + hook_file = arg + + check_zap_client_version() + + load_custom_hooks(hook_file) + trigger_hook('cli_opts', opts) # Check target supplied and ok if len(target) == 0: @@ -199,7 +231,7 @@ def main(argv): if running_in_docker(): base_dir = '/zap/wrk/' - if config_file or generate or report_html or report_xml or progress_file or context_file or target_file: + if config_file or generate or report_html or report_xml or report_json or progress_file or context_file or target_file: # Check directory has been mounted if not os.path.exists(base_dir): logging.warning('A file based option has been specified but the directory \'/zap/wrk\' is not mounted ') @@ -227,11 +259,19 @@ def main(argv): if config_file: # load config file from filestore with open(base_dir + config_file) as f: - load_config(f, config_dict, config_msg, out_of_scope_dict) + try: + load_config(f, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to load config file " + base_dir + config_file + " " + str(e)) + sys.exit(3) elif config_url: # load config file from url try: - load_config(urllib2.urlopen(config_url), config_dict, config_msg, out_of_scope_dict) + config_data = urlopen(config_url).read().decode('UTF-8').splitlines() + load_config(config_data, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to read configs from " + config_url + " " + str(e)) + sys.exit(3) except: logging.warning('Failed to read configs from ' + config_url) sys.exit(3) @@ -256,9 +296,7 @@ def main(argv): params.append('-addoninstall') params.append('pscanrulesAlpha') - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) start_zap(port, params) @@ -277,9 +315,7 @@ def main(argv): if (zap_alpha): params.extend(['-addoninstall', 'pscanrulesAlpha']) - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) try: cid = start_docker_zap('owasp/zap2docker-weekly', port, params, mount_dir) @@ -306,13 +342,12 @@ def main(argv): try: zap = ZAPv2(proxies={'http': 'http://' + zap_ip + ':' + str(port), 'https': 'http://' + zap_ip + ':' + str(port)}) - wait_for_zap_start(zap, timeout) + wait_for_zap_start(zap, timeout * 60) + trigger_hook('zap_started', zap, target) if context_file: # handle the context file, cant use base_dir as it might not have been set up - res = zap.context.import_context('/zap/wrk/' + os.path.basename(context_file)) - if res.startswith("ZAP Error"): - logging.error('Failed to load context file ' + context_file + ' : ' + res) + zap_import_context(zap, '/zap/wrk/' + os.path.basename(context_file)) # Enable scripts zap.script.load('Alert_on_HTTP_Response_Code_Errors.js', 'httpsender', 'Oracle Nashorn', '/home/zap/.ZAP_D/scripts/scripts/httpsender/Alert_on_HTTP_Response_Code_Errors.js') @@ -322,40 +357,50 @@ def main(argv): # Import the API defn if format == 'openapi': + trigger_hook('importing_openapi', target_url, target_file) if target_url: logging.debug('Import OpenAPI URL ' + target_url) - res = zap._request(zap.base + 'openapi/action/importUrl/', {'url':target}) - urls = zap.core.urls + res = zap.openapi.import_url(target, host_override) + urls = zap.core.urls() + if host_override: + target = urljoin(target_url, '//' + host_override) + logging.info('Using host override, new target: {0}'.format(target)) else: logging.debug('Import OpenAPI File ' + target_file) - res = zap._request(zap.base + 'openapi/action/importFile/', {'file': base_dir + target_file}) - urls = zap.core.urls + res = zap.openapi.import_file(base_dir + target_file) + urls = zap.core.urls() if len(urls) > 0: # Choose the first one - will be striping off the path below target = urls[0] - else: - logging.error('Failed to import any URLs') + logging.debug('Using target from imported file: {0}'.format(target)) else: + trigger_hook('importing_soap', target_url, target_file) if target_url: logging.debug('Import SOAP URL ' + target_url) res = zap._request(zap.base + 'soap/action/importUrl/', {'url':target}) - urls = zap.core.urls + urls = zap.core.urls() else: logging.debug('Import SOAP File ' + target_file) res = zap._request(zap.base + 'soap/action/importFile/', {'file': base_dir + target_file}) - urls = zap.core.urls + urls = zap.core.urls() if len(urls) > 0: # Choose the first one - will be striping off the path below target = urls[0] - else: - logging.error('Failed to import any URLs') + logging.debug('Using target from imported file: {0}'.format(target)) logging.info('Number of Imported URLs: ' + str(len(urls))) logging.debug('Import warnings: ' + str(res)) + if len(urls) == 0: + logging.warning('Failed to import any URLs') + # No point continue, there's nothing to scan. + raise NoUrlsException() + if target.count('/') > 2: + old_target = target # The url can include a valid path, but always reset to scan the host target = target[0:target.index('/', 8)+1] + logging.debug('Normalised target from {0} to {1}'.format(old_target, target)) # Wait for a delay if specified with -D option if (delay): @@ -376,12 +421,13 @@ def main(argv): # Dont bother checking the result - this will fail for pscan rules zap.ascan.set_scanner_alert_threshold(id=scanner, alertthreshold='OFF', scanpolicyname=scan_policy) - zap_active_scan(zap, target, scan_policy) + if not baseline: + zap_active_scan(zap, target, scan_policy) - zap_wait_for_passive_scan(zap) + zap_wait_for_passive_scan(zap, timeout * 60) # Print out a count of the number of urls - num_urls = len(zap.core.urls) + num_urls = len(zap.core.urls()) if num_urls == 0: logging.warning('No URLs found - is the target URL accessible? Local services may not be accessible from the Docker container') else: @@ -430,7 +476,7 @@ def main(argv): if not alert_dict.has_key(plugin_id) and not(config_dict.has_key(plugin_id) and config_dict[plugin_id] == 'IGNORE'): pass_dict[plugin_id] = rule.get('name') - if min_level == levels.index("PASS") and detailed_output: + if min_level == zap_conf_lvls.index("PASS") and detailed_output: for key, rule in sorted(pass_dict.iteritems()): print('PASS: ' + rule + ' [' + key + ']') @@ -446,40 +492,42 @@ def main(argv): print('SKIP: ' + rule.get('name') + ' [' + plugin_id + ']') # print out the ignored rules - ignore_count, not_used = print_rules(alert_dict, 'IGNORE', config_dict, config_msg, min_level, levels, + ignore_count, not_used = print_rules(zap, alert_dict, 'IGNORE', config_dict, config_msg, min_level, inc_ignore_rules, True, detailed_output, {}) # print out the info rules - info_count, not_used = print_rules(alert_dict, 'INFO', config_dict, config_msg, min_level, levels, + info_count, not_used = print_rules(zap, alert_dict, 'INFO', config_dict, config_msg, min_level, inc_info_rules, info_unspecified, detailed_output, in_progress_issues) # print out the warning rules - warn_count, warn_inprog_count = print_rules(alert_dict, 'WARN', config_dict, config_msg, min_level, levels, + warn_count, warn_inprog_count = print_rules(zap, alert_dict, 'WARN', config_dict, config_msg, min_level, inc_warn_rules, not info_unspecified, detailed_output, in_progress_issues) # print out the failing rules - fail_count, fail_inprog_count = print_rules(alert_dict, 'FAIL', config_dict, config_msg, min_level, levels, + fail_count, fail_inprog_count = print_rules(zap, alert_dict, 'FAIL', config_dict, config_msg, min_level, inc_fail_rules, True, detailed_output, in_progress_issues) if report_html: # Save the report - with open(base_dir + report_html, 'w') as f: - f.write(zap.core.htmlreport()) + write_report(base_dir + report_html, zap.core.htmlreport()) + + if report_json: + # Save the report + write_report(base_dir + report_json, zap.core.jsonreport()) if report_md: # Save the report - with open(base_dir + report_md, 'w') as f: - f.write(zap.core.mdreport()) + write_report(base_dir + report_md, zap.core.mdreport()) if report_xml: # Save the report - with open(base_dir + report_xml, 'w') as f: - f.write(zap.core.xmlreport()) + write_report(base_dir + report_xml, zap.core.xmlreport()) print('FAIL-NEW: ' + str(fail_count) + '\tFAIL-INPROG: ' + str(fail_inprog_count) + '\tWARN-NEW: ' + str(warn_count) + '\tWARN-INPROG: ' + str(warn_inprog_count) + '\tINFO: ' + str(info_count) + '\tIGNORE: ' + str(ignore_count) + '\tPASS: ' + str(pass_count)) + trigger_hook('zap_pre_shutdown', zap) # Stop ZAP zap.core.shutdown() @@ -491,7 +539,10 @@ def main(argv): else: print("ERROR %s" % e) logging.warning('I/O error: ' + str(e)) - dump_log_file(cid) + dump_log_file(cid) + + except NoUrlsException: + dump_log_file(cid) except: print("ERROR " + str(sys.exc_info()[0])) @@ -501,6 +552,8 @@ def main(argv): if not running_in_docker(): stop_docker(cid) + trigger_hook('pre_exit', fail_count, warn_count, pass_count) + if fail_count > 0: sys.exit(1) elif warn_count > 0: diff --git a/cicd/jenkins-slave-zap/centos/zap/zap-baseline.py b/cicd/jenkins-slave-zap/centos/zap/zap-baseline.py index b145d4e..4b83992 100755 --- a/cicd/jenkins-slave-zap/centos/zap/zap-baseline.py +++ b/cicd/jenkins-slave-zap/centos/zap/zap-baseline.py @@ -48,20 +48,17 @@ import os.path import sys import time -from six.moves.urllib.request import urlopen - from datetime import datetime from zapv2 import ZAPv2 from zap_common import * -timeout = 120 + config_dict = {} config_msg = {} out_of_scope_dict = {} -levels = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] min_level = 0 -# Pscan rules that aren't really relevant, eg the examples rules in the alpha set +# Pscan rules that aren't really relevant, e.g. the examples rules in the alpha set blacklist = ['-1', '50003', '60000', '60001'] # Pscan rules that are being addressed @@ -73,33 +70,37 @@ def usage(): - print ('Usage: zap-baseline.py -t [options]') - print (' -t target target URL including the protocol, eg https://www.example.com') - print ('Options:') - print (' -c config_file config file to use to INFO, IGNORE or FAIL warnings') - print (' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') - print (' -g gen_file generate default config file (all rules set to WARN)') - print (' -m mins the number of minutes to spider for (default 1)') - print (' -r report_html file to write the full ZAP HTML report') - print (' -w report_md file to write the full ZAP Wiki (Markdown) report') - print (' -x report_xml file to write the full ZAP XML report') - print (' -a include the alpha passive scan rules as well') - print (' -d show debug messages') - print (' -P specify listen port') - print (' -D delay in seconds to wait for passive scanning ') - print (' -i default rules not in the config file to INFO') - print (' -j use the Ajax spider in addition to the traditional one') - print (' -l level minimum level to show: PASS, IGNORE, INFO, WARN or FAIL, use with -s to hide example URLs') - print (' -n context_file context file which will be loaded prior to spidering the target') - print (' -p progress_file progress file which specifies issues that are being addressed') - print (' -s short output format - dont show PASSes or example URLs') - print (' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') - print ('') - print ('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan') + print('Usage: zap-baseline.py -t [options]') + print(' -t target target URL including the protocol, e.g. https://www.example.com') + print('Options:') + print(' -h print this help message') + print(' -c config_file config file to use to INFO, IGNORE or FAIL warnings') + print(' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') + print(' -g gen_file generate default config file (all rules set to WARN)') + print(' -m mins the number of minutes to spider for (default 1)') + print(' -r report_html file to write the full ZAP HTML report') + print(' -w report_md file to write the full ZAP Wiki (Markdown) report') + print(' -x report_xml file to write the full ZAP XML report') + print(' -J report_json file to write the full ZAP JSON document') + print(' -a include the alpha passive scan rules as well') + print(' -d show debug messages') + print(' -P specify listen port') + print(' -D delay in seconds to wait for passive scanning ') + print(' -i default rules not in the config file to INFO') + print(' -I do not return failure on warning') + print(' -j use the Ajax spider in addition to the traditional one') + print(' -l level minimum level to show: PASS, IGNORE, INFO, WARN or FAIL, use with -s to hide example URLs') + print(' -n context_file context file which will be loaded prior to spidering the target') + print(' -p progress_file progress file which specifies issues that are being addressed') + print(' -s short output format - dont show PASSes or example URLs') + print(' -T max time in minutes to wait for ZAP to start and the passive scan to run') + print(' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') + print(' --hook path to python file that define your custom hooks') + print('') + print('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan') def main(argv): - global min_level global in_progress_issues cid = '' @@ -114,6 +115,7 @@ def main(argv): report_html = '' report_md = '' report_xml = '' + report_json = '' target = '' zap_alpha = False info_unspecified = False @@ -122,6 +124,9 @@ def main(argv): zap_ip = 'localhost' zap_options = '' delay = 0 + timeout = 0 + ignore_warn = False + hook_file = None pass_count = 0 warn_count = 0 @@ -132,14 +137,17 @@ def main(argv): fail_inprog_count = 0 try: - opts, args = getopt.getopt(argv, "t:c:u:g:m:n:r:w:x:l:daijp:sz:P:D:") + opts, args = getopt.getopt(argv, "t:c:u:g:m:n:r:J:w:x:l:hdaijp:sz:P:D:T:I", ["hook="]) except getopt.GetoptError as exc: logging.warning('Invalid option ' + exc.opt + ' : ' + exc.msg) usage() sys.exit(3) for opt, arg in opts: - if opt == '-t': + if opt == '-h': + usage() + sys.exit(0) + elif opt == '-t': target = arg logging.debug('Target: ' + target) elif opt == '-c': @@ -162,6 +170,8 @@ def main(argv): progress_file = arg elif opt == '-r': report_html = arg + elif opt == '-J': + report_json = arg elif opt == '-w': report_md = arg elif opt == '-x': @@ -170,20 +180,30 @@ def main(argv): zap_alpha = True elif opt == '-i': info_unspecified = True + elif opt == '-I': + ignore_warn = True elif opt == '-j': ajax = True elif opt == '-l': try: - min_level = levels.index(arg) + min_level = zap_conf_lvls.index(arg) except ValueError: - logging.warning('Level must be one of ' + str(levels)) + logging.warning('Level must be one of ' + str(zap_conf_lvls)) usage() sys.exit(3) elif opt == '-z': zap_options = arg - elif opt == '-s': detailed_output = False + elif opt == '-T': + timeout = int(arg) + elif opt == '--hook': + hook_file = arg + + check_zap_client_version() + + load_custom_hooks(hook_file) + trigger_hook('cli_opts', opts) # Check target supplied and ok if len(target) == 0: @@ -197,7 +217,7 @@ def main(argv): if running_in_docker(): base_dir = '/zap/wrk/' - if config_file or generate or report_html or report_xml or progress_file or context_file: + if config_file or generate or report_html or report_xml or report_json or progress_file or context_file: # Check directory has been mounted if not os.path.exists(base_dir): logging.warning('A file based option has been specified but the directory \'/zap/wrk\' is not mounted ') @@ -213,11 +233,19 @@ def main(argv): if config_file: # load config file from filestore with open(base_dir + config_file) as f: - load_config(f, config_dict, config_msg, out_of_scope_dict) + try: + load_config(f, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to load config file " + base_dir + config_file + " " + str(e)) + sys.exit(3) elif config_url: # load config file from url try: - load_config(urlopen(config_url).read().decode('UTF-8'), config_dict, config_msg, out_of_scope_dict) + config_data = urlopen(config_url).read().decode('UTF-8').splitlines() + load_config(config_data, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to read configs from " + config_url + " " + str(e)) + sys.exit(3) except: logging.warning('Failed to read configs from ' + config_url) sys.exit(3) @@ -243,9 +271,7 @@ def main(argv): params.append('-addoninstall') params.append('pscanrulesAlpha') - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) start_zap(port, params) @@ -266,9 +292,7 @@ def main(argv): if (zap_alpha): params.extend(['-addoninstall', 'pscanrulesAlpha']) - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) try: cid = start_docker_zap('owasp/zap2docker-weekly', port, params, mount_dir) @@ -281,19 +305,14 @@ def main(argv): try: zap = ZAPv2(proxies={'http': 'http://' + zap_ip + ':' + str(port), 'https': 'http://' + zap_ip + ':' + str(port)}) - wait_for_zap_start(zap, timeout) + wait_for_zap_start(zap, timeout * 60) + trigger_hook('zap_started', zap, target) if context_file: # handle the context file, cant use base_dir as it might not have been set up - res = zap.context.import_context('/zap/wrk/' + os.path.basename(context_file)) - if res.startswith("ZAP Error"): - logging.error('Failed to load context file ' + context_file + ' : ' + res) + zap_import_context(zap, '/zap/wrk/' + os.path.basename(context_file)) - # Access the target - res = zap.urlopen(target) - if res.startswith("ZAP Error"): - # errno.EIO is 5, not sure why my atempts to import it failed;) - raise IOError(5, 'Failed to connect') + zap_access_target(zap, target) if target.count('/') > 2: # The url can include a valid path, but always reset to spider the host @@ -313,10 +332,10 @@ def main(argv): time.sleep(5) logging.debug('Delay passive scan check ' + str(delay - (datetime.now() - start_scan).seconds) + ' seconds') - zap_wait_for_passive_scan(zap) + zap_wait_for_passive_scan(zap, timeout * 60) # Print out a count of the number of urls - num_urls = len(zap.core.urls) + num_urls = len(zap.core.urls()) if num_urls == 0: logging.warning('No URLs found - is the target URL accessible? Local services may not be accessible from the Docker container') else: @@ -352,47 +371,49 @@ def main(argv): if (plugin_id not in alert_dict): pass_dict[plugin_id] = rule.get('name') - if min_level == levels.index("PASS") and detailed_output: + if min_level == zap_conf_lvls.index("PASS") and detailed_output: for key, rule in sorted(pass_dict.items()): print('PASS: ' + rule + ' [' + key + ']') pass_count = len(pass_dict) # print out the ignored rules - ignore_count, not_used = print_rules(alert_dict, 'IGNORE', config_dict, config_msg, min_level, levels, + ignore_count, not_used = print_rules(zap, alert_dict, 'IGNORE', config_dict, config_msg, min_level, inc_ignore_rules, True, detailed_output, {}) # print out the info rules - info_count, not_used = print_rules(alert_dict, 'INFO', config_dict, config_msg, min_level, levels, + info_count, not_used = print_rules(zap, alert_dict, 'INFO', config_dict, config_msg, min_level, inc_info_rules, info_unspecified, detailed_output, in_progress_issues) # print out the warning rules - warn_count, warn_inprog_count = print_rules(alert_dict, 'WARN', config_dict, config_msg, min_level, levels, + warn_count, warn_inprog_count = print_rules(zap, alert_dict, 'WARN', config_dict, config_msg, min_level, inc_warn_rules, not info_unspecified, detailed_output, in_progress_issues) # print out the failing rules - fail_count, fail_inprog_count = print_rules(alert_dict, 'FAIL', config_dict, config_msg, min_level, levels, + fail_count, fail_inprog_count = print_rules(zap, alert_dict, 'FAIL', config_dict, config_msg, min_level, inc_fail_rules, True, detailed_output, in_progress_issues) if report_html: # Save the report - with open(base_dir + report_html, 'w') as f: - f.write(zap.core.htmlreport()) + write_report(base_dir + report_html, zap.core.htmlreport()) + + if report_json: + # Save the report + write_report(base_dir + report_json, zap.core.jsonreport()) if report_md: # Save the report - with open(base_dir + report_md, 'w') as f: - f.write(zap.core.mdreport()) + write_report(base_dir + report_md, zap.core.mdreport()) if report_xml: # Save the report - with open(base_dir + report_xml, 'w') as f: - f.write(zap.core.xmlreport()) + write_report(base_dir + report_xml, zap.core.xmlreport()) print('FAIL-NEW: ' + str(fail_count) + '\tFAIL-INPROG: ' + str(fail_inprog_count) + '\tWARN-NEW: ' + str(warn_count) + '\tWARN-INPROG: ' + str(warn_inprog_count) + '\tINFO: ' + str(info_count) + '\tIGNORE: ' + str(ignore_count) + '\tPASS: ' + str(pass_count)) + trigger_hook('zap_pre_shutdown', zap) # Stop ZAP zap.core.shutdown() @@ -404,7 +425,7 @@ def main(argv): else: print("ERROR %s" % e) logging.warning('I/O error: ' + str(e)) - dump_log_file(cid) + dump_log_file(cid) except: print("ERROR " + str(sys.exc_info()[0])) @@ -414,9 +435,11 @@ def main(argv): if not running_in_docker(): stop_docker(cid) + trigger_hook('pre_exit', fail_count, warn_count, pass_count) + if fail_count > 0: sys.exit(1) - elif warn_count > 0: + elif (not ignore_warn) and warn_count > 0: sys.exit(2) elif pass_count > 0: sys.exit(0) diff --git a/cicd/jenkins-slave-zap/centos/zap/zap-full-scan.py b/cicd/jenkins-slave-zap/centos/zap/zap-full-scan.py index b488b51..3435dd8 100755 --- a/cicd/jenkins-slave-zap/centos/zap/zap-full-scan.py +++ b/cicd/jenkins-slave-zap/centos/zap/zap-full-scan.py @@ -41,7 +41,7 @@ # to be handled differently. # You can also add your own messages for the rules by appending them after a tab # at the end of each line. -# By default all of the active scan rules run but you can prevent rules from +# By default all of the active scan rules run but you can prevent rules from # running by supplying a configuration file with the rules set to IGNORE. import getopt @@ -51,20 +51,18 @@ import os.path import sys import time -import urllib2 from datetime import datetime from zapv2 import ZAPv2 from zap_common import * -timeout = 120 + config_dict = {} config_msg = {} out_of_scope_dict = {} -levels = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] min_level = 0 -# Scan rules that aren't really relevant, eg the examples rules in the alpha set -blacklist = ['-1', '50003', '60000', '60001'] +# Scan rules that aren't really relevant, e.g. the examples rules in the alpha set +blacklist = ['-1', '50003', '60000', '60001', '60100', '60101'] # Scan rules that are being addressed in_progress_issues = {} @@ -76,16 +74,18 @@ def usage(): print('Usage: zap-full-scan.py -t [options]') - print(' -t target target URL including the protocol, eg https://www.example.com') + print(' -t target target URL including the protocol, e.g. https://www.example.com') print('Options:') + print(' -h print this help message') print(' -c config_file config file to use to INFO, IGNORE or FAIL warnings') print(' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') print(' -g gen_file generate default config file(all rules set to WARN)') - print(' -m mins the number of minutes to spider for (default 1)') + print(' -m mins the number of minutes to spider for (defaults to no limit)') print(' -r report_html file to write the full ZAP HTML report') print(' -w report_md file to write the full ZAP Wiki(Markdown) report') print(' -x report_xml file to write the full ZAP XML report') - print(' -a include the alpha passive scan rules as well') + print(' -J report_json file to write the full ZAP JSON document') + print(' -a include the alpha active and passive scan rules as well') print(' -d show debug messages') print(' -P specify listen port') print(' -D delay in seconds to wait for passive scanning ') @@ -95,7 +95,9 @@ def usage(): print(' -n context_file context file which will be loaded prior to scanning the target') print(' -p progress_file progress file which specifies issues that are being addressed') print(' -s short output format - dont show PASSes or example URLs') + print(' -T max time in minutes to wait for ZAP to start and the passive scan to run') print(' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') + print(' --hook path to python file that define your custom hooks') print('') print('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan') @@ -116,6 +118,7 @@ def main(argv): report_html = '' report_md = '' report_xml = '' + report_json = '' target = '' zap_alpha = False info_unspecified = False @@ -124,6 +127,8 @@ def main(argv): zap_ip = 'localhost' zap_options = '' delay = 0 + timeout = 0 + hook_file = None pass_count = 0 warn_count = 0 @@ -134,14 +139,17 @@ def main(argv): fail_inprog_count = 0 try: - opts, args = getopt.getopt(argv, "t:c:u:g:m:n:r:w:x:l:daijp:sz:P:D:") + opts, args = getopt.getopt(argv, "t:c:u:g:m:n:r:J:w:x:l:hdaijp:sz:P:D:T:", ["hook="]) except getopt.GetoptError as exc: logging.warning('Invalid option ' + exc.opt + ' : ' + exc.msg) usage() sys.exit(3) for opt, arg in opts: - if opt == '-t': + if opt == '-h': + usage() + sys.exit(0) + elif opt == '-t': target = arg logging.debug('Target: ' + target) elif opt == '-c': @@ -164,6 +172,8 @@ def main(argv): progress_file = arg elif opt == '-r': report_html = arg + elif opt == '-J': + report_json = arg elif opt == '-w': report_md = arg elif opt == '-x': @@ -176,15 +186,24 @@ def main(argv): ajax = True elif opt == '-l': try: - min_level = levels.index(arg) + min_level = zap_conf_lvls.index(arg) except ValueError: - logging.warning('Level must be one of ' + str(levels)) + logging.warning('Level must be one of ' + str(zap_conf_lvls)) usage() sys.exit(3) elif opt == '-z': zap_options = arg elif opt == '-s': detailed_output = False + elif opt == '-T': + timeout = int(arg) + elif opt == '--hook': + hook_file = arg + + check_zap_client_version() + + load_custom_hooks(hook_file) + trigger_hook('cli_opts', opts) # Check target supplied and ok if len(target) == 0: @@ -198,7 +217,7 @@ def main(argv): if running_in_docker(): base_dir = '/zap/wrk/' - if config_file or generate or report_html or report_xml or progress_file or context_file: + if config_file or generate or report_html or report_xml or report_json or progress_file or context_file: # Check directory has been mounted if not os.path.exists(base_dir): logging.warning('A file based option has been specified but the directory \'/zap/wrk\' is not mounted ') @@ -214,11 +233,19 @@ def main(argv): if config_file: # load config file from filestore with open(base_dir + config_file) as f: - load_config(f, config_dict, config_msg, out_of_scope_dict) + try: + load_config(f, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to load config file " + base_dir + config_file + " " + str(e)) + sys.exit(3) elif config_url: # load config file from url try: - load_config(urllib2.urlopen(config_url), config_dict, config_msg, out_of_scope_dict) + config_data = urlopen(config_url).read().decode('UTF-8').splitlines() + load_config(config_data, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to read configs from " + config_url + " " + str(e)) + sys.exit(3) except: logging.warning('Failed to read configs from ' + config_url) sys.exit(3) @@ -238,15 +265,14 @@ def main(argv): params = [ '-config', 'spider.maxDuration=' + str(mins), '-addonupdate', - '-addoninstall', 'pscanrulesBeta'] # In case we're running in the stable container + '-addoninstall', 'pscanrulesBeta', # In case we're running in the stable container + '-addoninstall', 'ascanrulesBeta'] if zap_alpha: - params.append('-addoninstall') - params.append('pscanrulesAlpha') + params.extend(['-addoninstall', 'pscanrulesAlpha']) + params.extend(['-addoninstall', 'ascanrulesAlpha']) - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) start_zap(port, params) @@ -263,14 +289,14 @@ def main(argv): params = [ '-config', 'spider.maxDuration=' + str(mins), '-addonupdate', - '-addoninstall', 'pscanrulesBeta'] # In case we're running in the stable container + '-addoninstall', 'pscanrulesBeta', # In case we're running in the stable container + '-addoninstall', 'ascanrulesBeta'] if (zap_alpha): params.extend(['-addoninstall', 'pscanrulesAlpha']) + params.extend(['-addoninstall', 'ascanrulesAlpha']) - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) try: cid = start_docker_zap('owasp/zap2docker-weekly', port, params, mount_dir) @@ -283,19 +309,14 @@ def main(argv): try: zap = ZAPv2(proxies={'http': 'http://' + zap_ip + ':' + str(port), 'https': 'http://' + zap_ip + ':' + str(port)}) - wait_for_zap_start(zap, timeout) + wait_for_zap_start(zap, timeout * 60) + trigger_hook('zap_started', zap, target) if context_file: # handle the context file, cant use base_dir as it might not have been set up - res = zap.context.import_context('/zap/wrk/' + os.path.basename(context_file)) - if res.startswith("ZAP Error"): - logging.error('Failed to load context file ' + context_file + ' : ' + res) + zap_import_context(zap, '/zap/wrk/' + os.path.basename(context_file)) - # Access the target - res = zap.urlopen(target) - if res.startswith("ZAP Error"): - # errno.EIO is 5, not sure why my atempts to import it failed;) - raise IOError(5, 'Failed to connect') + zap_access_target(zap, target) if target.count('/') > 2: # The url can include a valid path, but always reset to spider the host @@ -331,10 +352,10 @@ def main(argv): zap_active_scan(zap, target, scan_policy) - zap_wait_for_passive_scan(zap) + zap_wait_for_passive_scan(zap, timeout * 60) # Print out a count of the number of urls - num_urls = len(zap.core.urls) + num_urls = len(zap.core.urls()) if num_urls == 0: logging.warning('No URLs found - is the target URL accessible? Local services may not be accessible from the Docker container') else: @@ -383,7 +404,7 @@ def main(argv): if not alert_dict.has_key(plugin_id) and not(config_dict.has_key(plugin_id) and config_dict[plugin_id] == 'IGNORE'): pass_dict[plugin_id] = rule.get('name') - if min_level == levels.index("PASS") and detailed_output: + if min_level == zap_conf_lvls.index("PASS") and detailed_output: for key, rule in sorted(pass_dict.iteritems()): print('PASS: ' + rule + ' [' + key + ']') @@ -399,40 +420,42 @@ def main(argv): print('SKIP: ' + rule.get('name') + ' [' + plugin_id + ']') # print out the ignored rules - ignore_count, not_used = print_rules(alert_dict, 'IGNORE', config_dict, config_msg, min_level, levels, + ignore_count, not_used = print_rules(zap, alert_dict, 'IGNORE', config_dict, config_msg, min_level, inc_ignore_rules, True, detailed_output, {}) # print out the info rules - info_count, not_used = print_rules(alert_dict, 'INFO', config_dict, config_msg, min_level, levels, + info_count, not_used = print_rules(zap, alert_dict, 'INFO', config_dict, config_msg, min_level, inc_info_rules, info_unspecified, detailed_output, in_progress_issues) # print out the warning rules - warn_count, warn_inprog_count = print_rules(alert_dict, 'WARN', config_dict, config_msg, min_level, levels, + warn_count, warn_inprog_count = print_rules(zap, alert_dict, 'WARN', config_dict, config_msg, min_level, inc_warn_rules, not info_unspecified, detailed_output, in_progress_issues) # print out the failing rules - fail_count, fail_inprog_count = print_rules(alert_dict, 'FAIL', config_dict, config_msg, min_level, levels, + fail_count, fail_inprog_count = print_rules(zap, alert_dict, 'FAIL', config_dict, config_msg, min_level, inc_fail_rules, True, detailed_output, in_progress_issues) if report_html: # Save the report - with open(base_dir + report_html, 'w') as f: - f.write(zap.core.htmlreport()) + write_report(base_dir + report_html, zap.core.htmlreport()) + + if report_json: + # Save the report + write_report(base_dir + report_json, zap.core.jsonreport()) if report_md: # Save the report - with open(base_dir + report_md, 'w') as f: - f.write(zap.core.mdreport()) + write_report(base_dir + report_md, zap.core.mdreport()) if report_xml: # Save the report - with open(base_dir + report_xml, 'w') as f: - f.write(zap.core.xmlreport()) + write_report(base_dir + report_xml, zap.core.xmlreport()) print('FAIL-NEW: ' + str(fail_count) + '\tFAIL-INPROG: ' + str(fail_inprog_count) + '\tWARN-NEW: ' + str(warn_count) + '\tWARN-INPROG: ' + str(warn_inprog_count) + '\tINFO: ' + str(info_count) + '\tIGNORE: ' + str(ignore_count) + '\tPASS: ' + str(pass_count)) + trigger_hook('zap_pre_shutdown', zap) # Stop ZAP zap.core.shutdown() @@ -444,7 +467,7 @@ def main(argv): else: print("ERROR %s" % e) logging.warning('I/O error: ' + str(e)) - dump_log_file(cid) + dump_log_file(cid) except: print("ERROR " + str(sys.exc_info()[0])) @@ -454,6 +477,8 @@ def main(argv): if not running_in_docker(): stop_docker(cid) + trigger_hook('pre_exit', fail_count, warn_count, pass_count) + if fail_count > 0: sys.exit(1) elif warn_count > 0: diff --git a/cicd/jenkins-slave-zap/centos/zap/zap-webswing.sh b/cicd/jenkins-slave-zap/centos/zap/zap-webswing.sh index 49c461d..d873b66 100755 --- a/cicd/jenkins-slave-zap/centos/zap/zap-webswing.sh +++ b/cicd/jenkins-slave-zap/centos/zap/zap-webswing.sh @@ -1,42 +1,41 @@ #!/bin/sh # # Startup script for the Webswing -# +# # Customised for ZAP running in Docker - just call with no parameters for webswing to start and leave # the docker container running # # Set environment. -export HOME=/zap/webswing-2.3/ -export OPTS="-h 0.0.0.0 -j $HOME/jetty.properties -u $HOME/user.properties -c $HOME/webswing.config" -export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +export HOME=/zap/webswing +export OPTS="-h 0.0.0.0 -j $HOME/jetty.properties -c $HOME/webswing.config" export JAVA_OPTS="-Xmx128M" export LOG=$HOME/webswing.out export PID_PATH_NAME=$HOME/webswing.pid -if [ -z `command -v $0` ]; then +if [ -z `command -v $0` ]; then CURRENTDIR=`pwd` cd `dirname $0` > /dev/null SCRIPTPATH=`pwd`/ cd $CURRENTDIR else - SCRIPTPATH="" + SCRIPTPATH="" fi if [ ! -f $HOME/webswing-server.war ]; then - echo "Webswing executable not found in $HOME folder" + echo "Webswing executable not found in $HOME folder" exit 1 fi if [ ! -f $JAVA_HOME/bin/java ]; then - echo "Java installation not found in $JAVA_HOME folder" + echo "Java installation not found in $JAVA_HOME folder" exit 1 fi if [ -z `command -v xvfb-run` ]; then - echo "Unable to locate xvfb-run command. Please install Xvfb before starting Webswing." + echo "Unable to locate xvfb-run command. Please install Xvfb before starting Webswing." exit 1 fi if [ ! -z `command -v ldconfig` ]; then - if [ `ldconfig -p | grep -i libXext | wc -l` -lt 1 ]; then + if [ `ldconfig -p | grep -i libXext | wc -l` -lt 1 ]; then echo "Missing dependent library libXext." exit 1 fi @@ -65,4 +64,4 @@ case "$1" in xvfb-run $SCRIPTPATH$0 run esac -exit 0 \ No newline at end of file +exit 0 diff --git a/cicd/jenkins-slave-zap/centos/zap/zap-x.sh b/cicd/jenkins-slave-zap/centos/zap/zap-x.sh index cb1a027..912ae09 100755 --- a/cicd/jenkins-slave-zap/centos/zap/zap-x.sh +++ b/cicd/jenkins-slave-zap/centos/zap/zap-x.sh @@ -4,4 +4,8 @@ if [ ! -f /tmp/.X1-lock ] then Xvfb :1 -screen 0 1024x768x16 -ac & fi -/zap/zap.sh $@ +/zap/zap.sh "$@" + +# Shutdown xvfb +kill -9 `cat /tmp/.X1-lock` +rm -f /tmp/.X1-lock diff --git a/cicd/jenkins-slave-zap/centos/zap/zap_common.py b/cicd/jenkins-slave-zap/centos/zap/zap_common.py index f7cb7d8..958fa3c 100755 --- a/cicd/jenkins-slave-zap/centos/zap/zap_common.py +++ b/cicd/jenkins-slave-zap/centos/zap/zap_common.py @@ -23,15 +23,104 @@ import logging import os import re +import shlex import socket import subprocess import sys import time import traceback import errno +import imp +import zapv2 from random import randint +from six.moves.urllib.request import urlopen +from six import binary_type +try: + import pkg_resources +except ImportError: + # don't hard fail since it's just used for the version check + logging.warning('Error importing pkg_resources. Is setuptools installed?') + +OLD_ZAP_CLIENT_WARNING = '''A newer version of python_owasp_zap_v2.4 + is available. Please run \'pip install -U python_owasp_zap_v2.4\' to update to + the latest version.'''.replace('\n', '') + +zap_conf_lvls = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] +zap_hooks = None + +def load_custom_hooks(hooks_file=None): + """ Loads a custom python module which modifies zap scripts behaviour + hooks_file - a python file which defines custom hooks + """ + global zap_hooks + hooks_file = hooks_file if hooks_file else os.environ.get('ZAP_HOOKS', '~/.zap_hooks.py') + hooks_file = os.path.expanduser(hooks_file) + + if not os.path.exists(hooks_file): + logging.debug('Could not find custom hooks file at %s ' % hooks_file) + return + + zap_hooks = imp.load_source("zap_hooks", hooks_file) + + +def hook(hook_name=None, **kwargs): + """ + Decorator method for calling hook before/after method. + Always adds a hook that runs before intercepting args and if wrap=True will create + another hook to intercept the return. + hook_name - name of hook for interactions, if None will use the name of the method it wrapped + """ + after_hook = kwargs.get('wrap', False) + def _decorator(func): + name = func.__name__ + _hook_name = hook_name if hook_name else name + def _wrap(*args, **kwargs): + hook_args = list(args) + hook_kwargs = dict(kwargs) + args = trigger_hook(_hook_name, *hook_args, **hook_kwargs) + args_list = list(args) + return_data = func(*args_list, **kwargs) + + if after_hook: + return trigger_hook('%s_wrap' % _hook_name, return_data, **hook_kwargs) + return return_data + return _wrap + return _decorator + + +def trigger_hook(name, *args, **kwargs): + """ Trigger execution of custom hook method if found """ + global zap_hooks + arg_length = len(args) + args_list = list(args) + args = args[0] if arg_length == 1 else args + + logging.debug('Trigger hook: %s, args: %s' % (name, arg_length)) + + if not zap_hooks: + return args + elif not hasattr(zap_hooks, name): + return args + + hook_fn = getattr(zap_hooks, name) + if not callable(hook_fn): + return args + + response = hook_fn(*args_list, **kwargs) + + # The number of args returned should match arguments passed + if not response: + return args + elif arg_length == 1: + return args + elif (isinstance(response, list) or isinstance(response, tuple)) and len(response) != arg_length: + return args + return response + + +@hook() def load_config(config, config_dict, config_msg, out_of_scope_dict): """ Loads the config file specified into: config_dict - a dictionary which maps plugin_ids to levels (IGNORE, WARN, FAIL) @@ -41,11 +130,13 @@ def load_config(config, config_dict, config_msg, out_of_scope_dict): for line in config: if not line.startswith('#') and len(line) > 1: (key, val, optional) = line.rstrip().split('\t', 2) - if key == 'OUTOFSCOPE': - for plugin_id in val.split(','): + if val == 'OUTOFSCOPE': + for plugin_id in key.split(','): if plugin_id not in out_of_scope_dict: out_of_scope_dict[plugin_id] = [] out_of_scope_dict[plugin_id].append(re.compile(optional)) + elif val not in zap_conf_lvls: + raise ValueError("Level {0} is not a supported level: {1}".format(val, zap_conf_lvls)) else: config_dict[key] = val if '\t' in optional: @@ -53,6 +144,7 @@ def load_config(config, config_dict, config_msg, out_of_scope_dict): config_msg[key] = usermsg else: config_msg[key] = '' + logging.debug('Loaded config: {0}'.format(config_dict)) def is_in_scope(plugin_id, url, out_of_scope_dict): @@ -74,7 +166,7 @@ def is_in_scope(plugin_id, url, out_of_scope_dict): return True -def print_rule(action, alert_list, detailed_output, user_msg, in_progress_issues): +def print_rule(zap, action, alert_list, detailed_output, user_msg, in_progress_issues): id = alert_list[0].get('pluginId') if id in in_progress_issues: print (action + '-IN_PROGRESS: ' + alert_list[0].get('alert') + ' [' + id + '] x ' + str(len(alert_list)) + ' ' + user_msg) @@ -83,13 +175,15 @@ def print_rule(action, alert_list, detailed_output, user_msg, in_progress_issues else: print (action + '-NEW: ' + alert_list[0].get('alert') + ' [' + id + '] x ' + str(len(alert_list)) + ' ' + user_msg) if detailed_output: - # Show (up to) first 5 urls + # Show (up to) first 5 urls, along with the response code (which we have to perform another request for) for alert in alert_list[0:5]: - print ('\t' + alert.get('url')) + msg = zap.core.message(alert.get('messageId')) + respHeader = msg['responseHeader'] + code = respHeader[respHeader.index(' ') + 1 : respHeader.index('\r\n')] + print ('\t' + alert.get('url') + ' (' + code + ')') -def print_rules(alert_dict, level, config_dict, config_msg, min_level, levels, inc_rule, inc_extra, detailed_output, in_progress_issues): - # print out the ignored rules +def print_rules(zap, alert_dict, level, config_dict, config_msg, min_level, inc_rule, inc_extra, detailed_output, in_progress_issues): count = 0 inprog_count = 0 for key, alert_list in sorted(alert_dict.items()): @@ -98,13 +192,13 @@ def print_rules(alert_dict, level, config_dict, config_msg, min_level, levels, i user_msg = '' if key in config_msg: user_msg = config_msg[key] - if min_level <= levels.index(level): - print_rule(level, alert_list, detailed_output, user_msg, in_progress_issues) + if min_level <= zap_conf_lvls.index(level): + print_rule(zap, level, alert_list, detailed_output, user_msg, in_progress_issues) if key in in_progress_issues: inprog_count += 1 else: count += 1 - return count, inprog_count + return trigger_hook('print_rules_wrap', count, inprog_count) def inc_ignore_rules(config_dict, key, inc_extra): @@ -149,6 +243,13 @@ def running_in_docker(): return os.path.exists('/.dockerenv') +def add_zap_options(params, zap_options): + if zap_options: + for zap_opt in shlex.split(zap_options): + params.append(zap_opt) + + +@hook() def start_zap(port, extra_zap_params): logging.debug('Starting ZAP') # All of the default common params @@ -160,13 +261,21 @@ def start_zap(port, extra_zap_params): '-config', 'api.addrs.addr.name=.*', '-config', 'api.addrs.addr.regex=true'] + params.extend(extra_zap_params) + + logging.info('Params: ' + str(params)) + with open('zap.out', "w") as outfile: - subprocess.Popen(params + extra_zap_params, stdout=outfile) + subprocess.Popen(params, stdout=outfile) -def wait_for_zap_start(zap, timeout): +def wait_for_zap_start(zap, timeout_in_secs = 600): version = None - for x in range(0, timeout): + if not timeout_in_secs: + # if ZAP doesn't start in 10 mins then its probably not going to start + timeout_in_secs = 600 + + for x in range(0, timeout_in_secs): try: version = zap.core.version logging.debug('ZAP Version ' + version) @@ -178,9 +287,10 @@ def wait_for_zap_start(zap, timeout): if not version: raise IOError( errno.EIO, - 'Failed to connect to ZAP after {0} seconds'.format(timeout)) + 'Failed to connect to ZAP after {0} seconds'.format(timeout_in_secs)) +@hook(wrap=True) def start_docker_zap(docker_image, port, extra_zap_params, mount_dir): try: logging.debug('Pulling ZAP Docker image: ' + docker_image) @@ -222,7 +332,7 @@ def get_free_port(): if not (sock.connect_ex(('127.0.0.1', port)) == 0): return port - + def ipaddress_for_cid(cid): insp_output = subprocess.check_output(['docker', 'inspect', cid]).strip().decode('utf-8') #logging.debug('Docker Inspect: ' + insp_output) @@ -248,6 +358,14 @@ def stop_docker(cid): logging.warning('Docker rm failed') +@hook() +def zap_access_target(zap, target): + res = zap.urlopen(target) + if res.startswith("ZAP Error"): + raise IOError(errno.EIO, 'ZAP failed to access: {0}'.format(target)) + + +@hook(wrap=True) def zap_spider(zap, target): logging.debug('Spider ' + target) spider_scan_id = zap.spider.scan(target) @@ -259,6 +377,7 @@ def zap_spider(zap, target): logging.debug('Spider complete') +@hook(wrap=True) def zap_ajax_spider(zap, target, max_time): logging.debug('AjaxSpider ' + target) if max_time: @@ -272,6 +391,7 @@ def zap_ajax_spider(zap, target, max_time): logging.debug('Ajax Spider complete') +@hook(wrap=True) def zap_active_scan(zap, target, policy): logging.debug('Active Scan ' + target + ' with policy ' + policy) ascan_scan_id = zap.ascan.scan(target, recurse=True, scanpolicyname=policy) @@ -284,15 +404,25 @@ def zap_active_scan(zap, target, policy): logging.debug(zap.ascan.scan_progress(ascan_scan_id)) -def zap_wait_for_passive_scan(zap): +def zap_wait_for_passive_scan(zap, timeout_in_secs = 0): rtc = zap.pscan.records_to_scan logging.debug('Records to scan...') + time_taken = 0 + timed_out = False while (int(zap.pscan.records_to_scan) > 0): logging.debug('Records to passive scan : ' + zap.pscan.records_to_scan) time.sleep(2) - logging.debug('Passive scanning complete') + time_taken += 2 + if timeout_in_secs and time_taken > timeout_in_secs: + timed_out = True + break + if timed_out: + logging.debug('Exceeded passive scan timeout') + else: + logging.debug('Passive scanning complete') +@hook(wrap=True) def zap_get_alerts(zap, baseurl, blacklist, out_of_scope_dict): # Retrieve the alerts using paging in case there are lots of them st = 0 @@ -319,3 +449,67 @@ def zap_get_alerts(zap, baseurl, blacklist, out_of_scope_dict): alerts = zap.core.alerts(start=st, count=pg) logging.debug('Total number of alerts: ' + str(alert_count)) return alert_dict + + +def get_latest_zap_client_version(): + version_info = None + + try: + version_info = urlopen('https://pypi.python.org/pypi/python-owasp-zap-v2.4/json', timeout=10) + except Exception as e: + logging.warning('Error fetching latest ZAP Python API client version: %s' % e) + return None + + if version_info is None: + logging.warning('Error fetching latest ZAP Python API client version: %s' % e) + return None + + version_json = json.loads(version_info.read().decode('utf-8')) + + if 'info' not in version_json: + logging.warning('No version found for latest ZAP Python API client.') + return None + if 'version' not in version_json['info']: + logging.warning('No version found for latest ZAP Python API client.') + return None + + return version_json['info']['version'] + + +def check_zap_client_version(): + # No need to check ZAP Python API client while running in Docker + if running_in_docker(): + return + + if 'pkg_resources' not in globals(): # import failed + logging.warning('Could not check ZAP Python API client without pkg_resources.') + return + + current_version = getattr(zapv2, '__version__', None) + latest_version = get_latest_zap_client_version() + parse_version = pkg_resources.parse_version + if current_version and latest_version and \ + parse_version(current_version) < parse_version(latest_version): + logging.warning(OLD_ZAP_CLIENT_WARNING) + elif current_version is None: + # the latest versions >= 0.0.9 have a __version__ + logging.warning(OLD_ZAP_CLIENT_WARNING) + # else: + # we're up to date or ahead / running a development build + # or latest_version is None and the user already got a warning + + +def write_report(file_path, report): + with open(file_path, mode='wb') as f: + if not isinstance(report, binary_type): + report = report.encode('utf-8') + + f.write(report) + +@hook(wrap=True) +def zap_import_context(zap, context_file): + res = context_id = zap.context.import_context(context_file) + if res.startswith("ZAP Error"): + context_id = None + logging.error('Failed to load context file ' + context_file + ' : ' + res) + return context_id diff --git a/cicd/jenkins-slave-zap/rhel7/Dockerfile b/cicd/jenkins-slave-zap/rhel7/Dockerfile index e9bc619..6983f94 100644 --- a/cicd/jenkins-slave-zap/rhel7/Dockerfile +++ b/cicd/jenkins-slave-zap/rhel7/Dockerfile @@ -21,49 +21,31 @@ LABEL summary="$SUMMARY" \ # We need to call 2 (!) yum commands before being able to enable repositories properly # This is a workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1479388 # Chrome install info: https://access.redhat.com/discussions/917293 -RUN yum repolist > /dev/null && \ - yum install -y yum-utils && \ - yum-config-manager --disable \* &> /dev/null && \ - yum-config-manager --enable rhel-server-rhscl-7-rpms && \ +RUN yum-config-manager --enable rhel-server-rhscl-7-rpms && \ yum-config-manager --enable rhel-7-server-rpms && \ yum-config-manager --enable rhel-7-server-extras-rpms && \ yum-config-manager --enable rhel-7-server-optional-rpms && \ yum-config-manager --enable rhel-7-server-fastrack-rpms && \ - yum-config-manager --enable epel && \ - yum clean all -y - # yum update -y - -RUN UNINSTALL_PKGS="java-1.8.0-openjdk-headless.i686 libstdc++-4.8.5-16.el7.i686" &&\ - yum remove -y $UNINSTALL_PKGS && \ - yum install -y redhat-rpm-config \ - make automake autoconf gcc gcc-c++ libstdc++ libstdc++-devel \ - java-1.8.0-openjdk.x86_64 java-1.8.0-openjdk-headless.x86_64 java-1.8.0-openjdk-devel.x86_64 \ - wget nano curl git gzip gettext tar xorg-x11-server-Xvfb xterm vnc-server \ - net-tools python27-python-pip firefox nss_wrapper && \ - yum -y clean all - -RUN wget https://pypi.python.org/packages/source/s/setuptools/setuptools-7.0.tar.gz --no-check-certificate && \ - tar xzf setuptools-7.0.tar.gz && \ - cd setuptools-7.0 && \ - python setup.py install && \ - wget https://bootstrap.pypa.io/get-pip.py && \ - python get-pip.py - -RUN pip install --upgrade --force-reinstall pip==9.0.3 && \ - pip install zapcli && \ - pip install python-owasp-zap-v2.4 - -COPY rpms/*.rpm ./ - -RUN yum -y localinstall xmlstarlet-1.6.1-1.el7.x86_64.rpm && \ - yum -y localinstall imlib2-1.4.5-9.el7.x86_64.rpm && \ - yum -y localinstall pyxdg-0.25-2.el7.nux.noarch.rpm && \ - yum -y localinstall openbox-libs-3.5.2-6.2.x86_64.rpm && \ - yum -y localinstall openbox-3.5.2-6.2.x86_64.rpm && \ - yum -y localinstall x11vnc-0.9.13-11.el7.x86_64.rpm && \ - yum -y clean all && \ - rm ./*.rpm && \ - if [ ! -d /tmp/.X11-unix ] ; then mkdir /tmp/.X11-unix; fi && \ + yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ + yum clean all -y && \ + yum update -y + +RUN yum install -y redhat-rpm-config \ + make automake autoconf gcc gcc-c++ \ + libstdc++ libstdc++-devel \ + wget curl git firefox \ + xmlstarlet gettext tar \ + x11vnc xorg-x11-server-Xvfb xterm \ + openbox net-tools nss_wrapper \ + python python-pip \ + java-11-openjdk-headless java-11-openjdk java-11-openjdk-devel && \ + yum clean all + +RUN pip install --upgrade pip +RUN pip install zapcli +RUN pip install python-owasp-zap-v2.4 + +RUN if [ ! -d /tmp/.X11-unix ] ; then mkdir /tmp/.X11-unix; fi && \ mkdir -p /zap/wrk && \ mkdir /zap@tmp && \ mkdir -p /var/lib/jenkins/.vnc @@ -78,18 +60,21 @@ ENV ZAP_PATH /zap/zap.sh ENV HOME /var/lib/jenkins # Default port for use with zapcli -ENV ZAP_PORT 8080 +ENV ZAP_PORT=8080 COPY policies /var/lib/jenkins/.ZAP/policies/ COPY .xinitrc /var/lib/jenkins/ WORKDIR /zap + +ENV WEBSWING_VERSION=2.7.1 + # Download and expand the latest stable release RUN curl -s https://raw.githubusercontent.com/zaproxy/zap-admin/master/ZapVersions-dev.xml | xmlstarlet sel -t -v //url |grep -i Linux | wget -q --content-disposition -i - -O - | tar zx --strip-components=1 && \ - curl -s -L https://bitbucket.org/meszarv/webswing/downloads/webswing-2.3-distribution.zip | jar -x && \ + curl -s -L https://bitbucket.org/meszarv/webswing/downloads/webswing-${WEBSWING_VERSION}.zip | jar -x && \ touch AcceptedLicense -ADD webswing.config /zap/webswing-2.3/webswing.config +ADD webswing.config /zap/webswing-${WEBSWING_VERSION}/webswing.config RUN chown -v -R root:root /zap && \ chown -v -R root:root /var/lib/jenkins && \ diff --git a/cicd/jenkins-slave-zap/rhel7/rpms/Readme.md b/cicd/jenkins-slave-zap/rhel7/rpms/Readme.md deleted file mode 100644 index cc0b875..0000000 --- a/cicd/jenkins-slave-zap/rhel7/rpms/Readme.md +++ /dev/null @@ -1,9 +0,0 @@ -RPMs required for install - - -ftp://ftp.pbone.net/mirror/download.fedora.redhat.com/pub/fedora/epel/7/x86_64/Packages/x/xmlstarlet-1.6.1-1.el7.x86_64.rpm - -ftp://ftp.pbone.net/mirror/ftp5.gwdg.de/pub/opensuse/repositories/home:/TheIndifferent:/rhel7-openbox/rhel7-shared/x86_64/openbox-3.5.2-6.2.x86_64.rpm - -ftp://ftp.pbone.net/mirror/download.fedora.redhat.com/pub/fedora/epel/7/x86_64/Packages/x/x11vnc-0.9.13-11.el7.x86_64.rpm - diff --git a/cicd/jenkins-slave-zap/rhel7/rpms/imlib2-1.4.5-9.el7.x86_64.rpm b/cicd/jenkins-slave-zap/rhel7/rpms/imlib2-1.4.5-9.el7.x86_64.rpm deleted file mode 100644 index 6983f12..0000000 Binary files a/cicd/jenkins-slave-zap/rhel7/rpms/imlib2-1.4.5-9.el7.x86_64.rpm and /dev/null differ diff --git a/cicd/jenkins-slave-zap/rhel7/rpms/openbox-3.5.2-6.2.x86_64.rpm b/cicd/jenkins-slave-zap/rhel7/rpms/openbox-3.5.2-6.2.x86_64.rpm deleted file mode 100644 index 0061f56..0000000 Binary files a/cicd/jenkins-slave-zap/rhel7/rpms/openbox-3.5.2-6.2.x86_64.rpm and /dev/null differ diff --git a/cicd/jenkins-slave-zap/rhel7/rpms/openbox-libs-3.5.2-6.2.x86_64.rpm b/cicd/jenkins-slave-zap/rhel7/rpms/openbox-libs-3.5.2-6.2.x86_64.rpm deleted file mode 100644 index 518f254..0000000 Binary files a/cicd/jenkins-slave-zap/rhel7/rpms/openbox-libs-3.5.2-6.2.x86_64.rpm and /dev/null differ diff --git a/cicd/jenkins-slave-zap/rhel7/rpms/pyxdg-0.25-2.el7.nux.noarch.rpm b/cicd/jenkins-slave-zap/rhel7/rpms/pyxdg-0.25-2.el7.nux.noarch.rpm deleted file mode 100644 index 376fdb3..0000000 Binary files a/cicd/jenkins-slave-zap/rhel7/rpms/pyxdg-0.25-2.el7.nux.noarch.rpm and /dev/null differ diff --git a/cicd/jenkins-slave-zap/rhel7/rpms/x11vnc-0.9.13-11.el7.x86_64.rpm b/cicd/jenkins-slave-zap/rhel7/rpms/x11vnc-0.9.13-11.el7.x86_64.rpm deleted file mode 100644 index afb2660..0000000 Binary files a/cicd/jenkins-slave-zap/rhel7/rpms/x11vnc-0.9.13-11.el7.x86_64.rpm and /dev/null differ diff --git a/cicd/jenkins-slave-zap/rhel7/rpms/xmlstarlet-1.6.1-1.el7.x86_64.rpm b/cicd/jenkins-slave-zap/rhel7/rpms/xmlstarlet-1.6.1-1.el7.x86_64.rpm deleted file mode 100644 index f2b7a11..0000000 Binary files a/cicd/jenkins-slave-zap/rhel7/rpms/xmlstarlet-1.6.1-1.el7.x86_64.rpm and /dev/null differ diff --git a/cicd/jenkins-slave-zap/rhel7/zap/zap-api-scan.py b/cicd/jenkins-slave-zap/rhel7/zap/zap-api-scan.py index b958910..4066e64 100755 --- a/cicd/jenkins-slave-zap/rhel7/zap/zap-api-scan.py +++ b/cicd/jenkins-slave-zap/rhel7/zap/zap-api-scan.py @@ -55,19 +55,22 @@ import subprocess import sys import time -import urllib2 from datetime import datetime +from six.moves.urllib.parse import urljoin from zapv2 import ZAPv2 from zap_common import * -timeout = 120 + +class NoUrlsException(Exception): + pass + + config_dict = {} config_msg = {} out_of_scope_dict = {} -levels = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] min_level = 0 -# Scan rules that aren't really relevant, eg the examples rules in the alpha set +# Scan rules that aren't really relevant, e.g. the examples rules in the alpha set blacklist = ['-1', '50003', '60000', '60001'] # Scan rules that are being addressed @@ -80,15 +83,17 @@ def usage(): print('Usage: zap-api-scan.py -t -f [options]') - print(' -t target target API definition, OpenAPI or SOAP, local file or URL, eg https://www.example.com/openapi.json') + print(' -t target target API definition, OpenAPI or SOAP, local file or URL, e.g. https://www.example.com/openapi.json') print(' -f format either openapi or soap') print('Options:') + print(' -h print this help message') print(' -c config_file config file to use to INFO, IGNORE or FAIL warnings') print(' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') print(' -g gen_file generate default config file(all rules set to WARN)') print(' -r report_html file to write the full ZAP HTML report') print(' -w report_md file to write the full ZAP Wiki(Markdown) report') print(' -x report_xml file to write the full ZAP XML report') + print(' -J report_json file to write the full ZAP JSON document') print(' -a include the alpha passive scan rules as well') print(' -d show debug messages') print(' -P specify listen port') @@ -98,7 +103,11 @@ def usage(): print(' -n context_file context file which will be loaded prior to scanning the target') print(' -p progress_file progress file which specifies issues that are being addressed') print(' -s short output format - dont show PASSes or example URLs') + print(' -S safe mode this will skip the active scan and perform a baseline scan') + print(' -T max time in minutes to wait for ZAP to start and the passive scan to run') + print(' -O the hostname to override in the (remote) OpenAPI spec') print(' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') + print(' --hook path to python file that define your custom hooks') print('') print('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-API-Scan') @@ -118,16 +127,21 @@ def main(argv): report_html = '' report_md = '' report_xml = '' + report_json = '' target = '' target_file = '' target_url = '' + host_override = '' format = '' zap_alpha = False + baseline = False info_unspecified = False base_dir = '' zap_ip = 'localhost' zap_options = '' delay = 0 + timeout = 0 + hook_file = None pass_count = 0 warn_count = 0 @@ -138,14 +152,17 @@ def main(argv): fail_inprog_count = 0 try: - opts, args = getopt.getopt(argv, "t:f:c:u:g:m:n:r:w:x:l:daijp:sz:P:D:") + opts, args = getopt.getopt(argv, "t:f:c:u:g:m:n:r:J:w:x:l:hdaijSp:sz:P:D:T:O:", ["hook="]) except getopt.GetoptError as exc: logging.warning('Invalid option ' + exc.opt + ' : ' + exc.msg) usage() sys.exit(3) for opt, arg in opts: - if opt == '-t': + if opt == '-h': + usage() + sys.exit(0) + elif opt == '-t': target = arg logging.debug('Target: ' + target) elif opt == '-f': @@ -168,6 +185,8 @@ def main(argv): progress_file = arg elif opt == '-r': report_html = arg + elif opt == '-J': + report_json = arg elif opt == '-w': report_md = arg elif opt == '-x': @@ -178,15 +197,28 @@ def main(argv): info_unspecified = True elif opt == '-l': try: - min_level = levels.index(arg) + min_level = zap_conf_lvls.index(arg) except ValueError: - logging.warning('Level must be one of ' + str(levels)) + logging.warning('Level must be one of ' + str(zap_conf_lvls)) usage() sys.exit(3) elif opt == '-z': zap_options = arg elif opt == '-s': detailed_output = False + elif opt == '-S': + baseline = True + elif opt == '-T': + timeout = int(arg) + elif opt == '-O': + host_override = arg + elif opt == '--hook': + hook_file = arg + + check_zap_client_version() + + load_custom_hooks(hook_file) + trigger_hook('cli_opts', opts) # Check target supplied and ok if len(target) == 0: @@ -199,7 +231,7 @@ def main(argv): if running_in_docker(): base_dir = '/zap/wrk/' - if config_file or generate or report_html or report_xml or progress_file or context_file or target_file: + if config_file or generate or report_html or report_xml or report_json or progress_file or context_file or target_file: # Check directory has been mounted if not os.path.exists(base_dir): logging.warning('A file based option has been specified but the directory \'/zap/wrk\' is not mounted ') @@ -227,11 +259,19 @@ def main(argv): if config_file: # load config file from filestore with open(base_dir + config_file) as f: - load_config(f, config_dict, config_msg, out_of_scope_dict) + try: + load_config(f, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to load config file " + base_dir + config_file + " " + str(e)) + sys.exit(3) elif config_url: # load config file from url try: - load_config(urllib2.urlopen(config_url), config_dict, config_msg, out_of_scope_dict) + config_data = urlopen(config_url).read().decode('UTF-8').splitlines() + load_config(config_data, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to read configs from " + config_url + " " + str(e)) + sys.exit(3) except: logging.warning('Failed to read configs from ' + config_url) sys.exit(3) @@ -256,9 +296,7 @@ def main(argv): params.append('-addoninstall') params.append('pscanrulesAlpha') - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) start_zap(port, params) @@ -277,9 +315,7 @@ def main(argv): if (zap_alpha): params.extend(['-addoninstall', 'pscanrulesAlpha']) - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) try: cid = start_docker_zap('owasp/zap2docker-weekly', port, params, mount_dir) @@ -306,13 +342,12 @@ def main(argv): try: zap = ZAPv2(proxies={'http': 'http://' + zap_ip + ':' + str(port), 'https': 'http://' + zap_ip + ':' + str(port)}) - wait_for_zap_start(zap, timeout) + wait_for_zap_start(zap, timeout * 60) + trigger_hook('zap_started', zap, target) if context_file: # handle the context file, cant use base_dir as it might not have been set up - res = zap.context.import_context('/zap/wrk/' + os.path.basename(context_file)) - if res.startswith("ZAP Error"): - logging.error('Failed to load context file ' + context_file + ' : ' + res) + zap_import_context(zap, '/zap/wrk/' + os.path.basename(context_file)) # Enable scripts zap.script.load('Alert_on_HTTP_Response_Code_Errors.js', 'httpsender', 'Oracle Nashorn', '/home/zap/.ZAP_D/scripts/scripts/httpsender/Alert_on_HTTP_Response_Code_Errors.js') @@ -322,40 +357,50 @@ def main(argv): # Import the API defn if format == 'openapi': + trigger_hook('importing_openapi', target_url, target_file) if target_url: logging.debug('Import OpenAPI URL ' + target_url) - res = zap._request(zap.base + 'openapi/action/importUrl/', {'url':target}) - urls = zap.core.urls + res = zap.openapi.import_url(target, host_override) + urls = zap.core.urls() + if host_override: + target = urljoin(target_url, '//' + host_override) + logging.info('Using host override, new target: {0}'.format(target)) else: logging.debug('Import OpenAPI File ' + target_file) - res = zap._request(zap.base + 'openapi/action/importFile/', {'file': base_dir + target_file}) - urls = zap.core.urls + res = zap.openapi.import_file(base_dir + target_file) + urls = zap.core.urls() if len(urls) > 0: # Choose the first one - will be striping off the path below target = urls[0] - else: - logging.error('Failed to import any URLs') + logging.debug('Using target from imported file: {0}'.format(target)) else: + trigger_hook('importing_soap', target_url, target_file) if target_url: logging.debug('Import SOAP URL ' + target_url) res = zap._request(zap.base + 'soap/action/importUrl/', {'url':target}) - urls = zap.core.urls + urls = zap.core.urls() else: logging.debug('Import SOAP File ' + target_file) res = zap._request(zap.base + 'soap/action/importFile/', {'file': base_dir + target_file}) - urls = zap.core.urls + urls = zap.core.urls() if len(urls) > 0: # Choose the first one - will be striping off the path below target = urls[0] - else: - logging.error('Failed to import any URLs') + logging.debug('Using target from imported file: {0}'.format(target)) logging.info('Number of Imported URLs: ' + str(len(urls))) logging.debug('Import warnings: ' + str(res)) + if len(urls) == 0: + logging.warning('Failed to import any URLs') + # No point continue, there's nothing to scan. + raise NoUrlsException() + if target.count('/') > 2: + old_target = target # The url can include a valid path, but always reset to scan the host target = target[0:target.index('/', 8)+1] + logging.debug('Normalised target from {0} to {1}'.format(old_target, target)) # Wait for a delay if specified with -D option if (delay): @@ -376,12 +421,13 @@ def main(argv): # Dont bother checking the result - this will fail for pscan rules zap.ascan.set_scanner_alert_threshold(id=scanner, alertthreshold='OFF', scanpolicyname=scan_policy) - zap_active_scan(zap, target, scan_policy) + if not baseline: + zap_active_scan(zap, target, scan_policy) - zap_wait_for_passive_scan(zap) + zap_wait_for_passive_scan(zap, timeout * 60) # Print out a count of the number of urls - num_urls = len(zap.core.urls) + num_urls = len(zap.core.urls()) if num_urls == 0: logging.warning('No URLs found - is the target URL accessible? Local services may not be accessible from the Docker container') else: @@ -430,7 +476,7 @@ def main(argv): if not alert_dict.has_key(plugin_id) and not(config_dict.has_key(plugin_id) and config_dict[plugin_id] == 'IGNORE'): pass_dict[plugin_id] = rule.get('name') - if min_level == levels.index("PASS") and detailed_output: + if min_level == zap_conf_lvls.index("PASS") and detailed_output: for key, rule in sorted(pass_dict.iteritems()): print('PASS: ' + rule + ' [' + key + ']') @@ -446,40 +492,42 @@ def main(argv): print('SKIP: ' + rule.get('name') + ' [' + plugin_id + ']') # print out the ignored rules - ignore_count, not_used = print_rules(alert_dict, 'IGNORE', config_dict, config_msg, min_level, levels, + ignore_count, not_used = print_rules(zap, alert_dict, 'IGNORE', config_dict, config_msg, min_level, inc_ignore_rules, True, detailed_output, {}) # print out the info rules - info_count, not_used = print_rules(alert_dict, 'INFO', config_dict, config_msg, min_level, levels, + info_count, not_used = print_rules(zap, alert_dict, 'INFO', config_dict, config_msg, min_level, inc_info_rules, info_unspecified, detailed_output, in_progress_issues) # print out the warning rules - warn_count, warn_inprog_count = print_rules(alert_dict, 'WARN', config_dict, config_msg, min_level, levels, + warn_count, warn_inprog_count = print_rules(zap, alert_dict, 'WARN', config_dict, config_msg, min_level, inc_warn_rules, not info_unspecified, detailed_output, in_progress_issues) # print out the failing rules - fail_count, fail_inprog_count = print_rules(alert_dict, 'FAIL', config_dict, config_msg, min_level, levels, + fail_count, fail_inprog_count = print_rules(zap, alert_dict, 'FAIL', config_dict, config_msg, min_level, inc_fail_rules, True, detailed_output, in_progress_issues) if report_html: # Save the report - with open(base_dir + report_html, 'w') as f: - f.write(zap.core.htmlreport()) + write_report(base_dir + report_html, zap.core.htmlreport()) + + if report_json: + # Save the report + write_report(base_dir + report_json, zap.core.jsonreport()) if report_md: # Save the report - with open(base_dir + report_md, 'w') as f: - f.write(zap.core.mdreport()) + write_report(base_dir + report_md, zap.core.mdreport()) if report_xml: # Save the report - with open(base_dir + report_xml, 'w') as f: - f.write(zap.core.xmlreport()) + write_report(base_dir + report_xml, zap.core.xmlreport()) print('FAIL-NEW: ' + str(fail_count) + '\tFAIL-INPROG: ' + str(fail_inprog_count) + '\tWARN-NEW: ' + str(warn_count) + '\tWARN-INPROG: ' + str(warn_inprog_count) + '\tINFO: ' + str(info_count) + '\tIGNORE: ' + str(ignore_count) + '\tPASS: ' + str(pass_count)) + trigger_hook('zap_pre_shutdown', zap) # Stop ZAP zap.core.shutdown() @@ -491,7 +539,10 @@ def main(argv): else: print("ERROR %s" % e) logging.warning('I/O error: ' + str(e)) - dump_log_file(cid) + dump_log_file(cid) + + except NoUrlsException: + dump_log_file(cid) except: print("ERROR " + str(sys.exc_info()[0])) @@ -501,6 +552,8 @@ def main(argv): if not running_in_docker(): stop_docker(cid) + trigger_hook('pre_exit', fail_count, warn_count, pass_count) + if fail_count > 0: sys.exit(1) elif warn_count > 0: diff --git a/cicd/jenkins-slave-zap/rhel7/zap/zap-baseline.py b/cicd/jenkins-slave-zap/rhel7/zap/zap-baseline.py index b145d4e..4b83992 100755 --- a/cicd/jenkins-slave-zap/rhel7/zap/zap-baseline.py +++ b/cicd/jenkins-slave-zap/rhel7/zap/zap-baseline.py @@ -48,20 +48,17 @@ import os.path import sys import time -from six.moves.urllib.request import urlopen - from datetime import datetime from zapv2 import ZAPv2 from zap_common import * -timeout = 120 + config_dict = {} config_msg = {} out_of_scope_dict = {} -levels = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] min_level = 0 -# Pscan rules that aren't really relevant, eg the examples rules in the alpha set +# Pscan rules that aren't really relevant, e.g. the examples rules in the alpha set blacklist = ['-1', '50003', '60000', '60001'] # Pscan rules that are being addressed @@ -73,33 +70,37 @@ def usage(): - print ('Usage: zap-baseline.py -t [options]') - print (' -t target target URL including the protocol, eg https://www.example.com') - print ('Options:') - print (' -c config_file config file to use to INFO, IGNORE or FAIL warnings') - print (' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') - print (' -g gen_file generate default config file (all rules set to WARN)') - print (' -m mins the number of minutes to spider for (default 1)') - print (' -r report_html file to write the full ZAP HTML report') - print (' -w report_md file to write the full ZAP Wiki (Markdown) report') - print (' -x report_xml file to write the full ZAP XML report') - print (' -a include the alpha passive scan rules as well') - print (' -d show debug messages') - print (' -P specify listen port') - print (' -D delay in seconds to wait for passive scanning ') - print (' -i default rules not in the config file to INFO') - print (' -j use the Ajax spider in addition to the traditional one') - print (' -l level minimum level to show: PASS, IGNORE, INFO, WARN or FAIL, use with -s to hide example URLs') - print (' -n context_file context file which will be loaded prior to spidering the target') - print (' -p progress_file progress file which specifies issues that are being addressed') - print (' -s short output format - dont show PASSes or example URLs') - print (' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') - print ('') - print ('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan') + print('Usage: zap-baseline.py -t [options]') + print(' -t target target URL including the protocol, e.g. https://www.example.com') + print('Options:') + print(' -h print this help message') + print(' -c config_file config file to use to INFO, IGNORE or FAIL warnings') + print(' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') + print(' -g gen_file generate default config file (all rules set to WARN)') + print(' -m mins the number of minutes to spider for (default 1)') + print(' -r report_html file to write the full ZAP HTML report') + print(' -w report_md file to write the full ZAP Wiki (Markdown) report') + print(' -x report_xml file to write the full ZAP XML report') + print(' -J report_json file to write the full ZAP JSON document') + print(' -a include the alpha passive scan rules as well') + print(' -d show debug messages') + print(' -P specify listen port') + print(' -D delay in seconds to wait for passive scanning ') + print(' -i default rules not in the config file to INFO') + print(' -I do not return failure on warning') + print(' -j use the Ajax spider in addition to the traditional one') + print(' -l level minimum level to show: PASS, IGNORE, INFO, WARN or FAIL, use with -s to hide example URLs') + print(' -n context_file context file which will be loaded prior to spidering the target') + print(' -p progress_file progress file which specifies issues that are being addressed') + print(' -s short output format - dont show PASSes or example URLs') + print(' -T max time in minutes to wait for ZAP to start and the passive scan to run') + print(' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') + print(' --hook path to python file that define your custom hooks') + print('') + print('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan') def main(argv): - global min_level global in_progress_issues cid = '' @@ -114,6 +115,7 @@ def main(argv): report_html = '' report_md = '' report_xml = '' + report_json = '' target = '' zap_alpha = False info_unspecified = False @@ -122,6 +124,9 @@ def main(argv): zap_ip = 'localhost' zap_options = '' delay = 0 + timeout = 0 + ignore_warn = False + hook_file = None pass_count = 0 warn_count = 0 @@ -132,14 +137,17 @@ def main(argv): fail_inprog_count = 0 try: - opts, args = getopt.getopt(argv, "t:c:u:g:m:n:r:w:x:l:daijp:sz:P:D:") + opts, args = getopt.getopt(argv, "t:c:u:g:m:n:r:J:w:x:l:hdaijp:sz:P:D:T:I", ["hook="]) except getopt.GetoptError as exc: logging.warning('Invalid option ' + exc.opt + ' : ' + exc.msg) usage() sys.exit(3) for opt, arg in opts: - if opt == '-t': + if opt == '-h': + usage() + sys.exit(0) + elif opt == '-t': target = arg logging.debug('Target: ' + target) elif opt == '-c': @@ -162,6 +170,8 @@ def main(argv): progress_file = arg elif opt == '-r': report_html = arg + elif opt == '-J': + report_json = arg elif opt == '-w': report_md = arg elif opt == '-x': @@ -170,20 +180,30 @@ def main(argv): zap_alpha = True elif opt == '-i': info_unspecified = True + elif opt == '-I': + ignore_warn = True elif opt == '-j': ajax = True elif opt == '-l': try: - min_level = levels.index(arg) + min_level = zap_conf_lvls.index(arg) except ValueError: - logging.warning('Level must be one of ' + str(levels)) + logging.warning('Level must be one of ' + str(zap_conf_lvls)) usage() sys.exit(3) elif opt == '-z': zap_options = arg - elif opt == '-s': detailed_output = False + elif opt == '-T': + timeout = int(arg) + elif opt == '--hook': + hook_file = arg + + check_zap_client_version() + + load_custom_hooks(hook_file) + trigger_hook('cli_opts', opts) # Check target supplied and ok if len(target) == 0: @@ -197,7 +217,7 @@ def main(argv): if running_in_docker(): base_dir = '/zap/wrk/' - if config_file or generate or report_html or report_xml or progress_file or context_file: + if config_file or generate or report_html or report_xml or report_json or progress_file or context_file: # Check directory has been mounted if not os.path.exists(base_dir): logging.warning('A file based option has been specified but the directory \'/zap/wrk\' is not mounted ') @@ -213,11 +233,19 @@ def main(argv): if config_file: # load config file from filestore with open(base_dir + config_file) as f: - load_config(f, config_dict, config_msg, out_of_scope_dict) + try: + load_config(f, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to load config file " + base_dir + config_file + " " + str(e)) + sys.exit(3) elif config_url: # load config file from url try: - load_config(urlopen(config_url).read().decode('UTF-8'), config_dict, config_msg, out_of_scope_dict) + config_data = urlopen(config_url).read().decode('UTF-8').splitlines() + load_config(config_data, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to read configs from " + config_url + " " + str(e)) + sys.exit(3) except: logging.warning('Failed to read configs from ' + config_url) sys.exit(3) @@ -243,9 +271,7 @@ def main(argv): params.append('-addoninstall') params.append('pscanrulesAlpha') - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) start_zap(port, params) @@ -266,9 +292,7 @@ def main(argv): if (zap_alpha): params.extend(['-addoninstall', 'pscanrulesAlpha']) - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) try: cid = start_docker_zap('owasp/zap2docker-weekly', port, params, mount_dir) @@ -281,19 +305,14 @@ def main(argv): try: zap = ZAPv2(proxies={'http': 'http://' + zap_ip + ':' + str(port), 'https': 'http://' + zap_ip + ':' + str(port)}) - wait_for_zap_start(zap, timeout) + wait_for_zap_start(zap, timeout * 60) + trigger_hook('zap_started', zap, target) if context_file: # handle the context file, cant use base_dir as it might not have been set up - res = zap.context.import_context('/zap/wrk/' + os.path.basename(context_file)) - if res.startswith("ZAP Error"): - logging.error('Failed to load context file ' + context_file + ' : ' + res) + zap_import_context(zap, '/zap/wrk/' + os.path.basename(context_file)) - # Access the target - res = zap.urlopen(target) - if res.startswith("ZAP Error"): - # errno.EIO is 5, not sure why my atempts to import it failed;) - raise IOError(5, 'Failed to connect') + zap_access_target(zap, target) if target.count('/') > 2: # The url can include a valid path, but always reset to spider the host @@ -313,10 +332,10 @@ def main(argv): time.sleep(5) logging.debug('Delay passive scan check ' + str(delay - (datetime.now() - start_scan).seconds) + ' seconds') - zap_wait_for_passive_scan(zap) + zap_wait_for_passive_scan(zap, timeout * 60) # Print out a count of the number of urls - num_urls = len(zap.core.urls) + num_urls = len(zap.core.urls()) if num_urls == 0: logging.warning('No URLs found - is the target URL accessible? Local services may not be accessible from the Docker container') else: @@ -352,47 +371,49 @@ def main(argv): if (plugin_id not in alert_dict): pass_dict[plugin_id] = rule.get('name') - if min_level == levels.index("PASS") and detailed_output: + if min_level == zap_conf_lvls.index("PASS") and detailed_output: for key, rule in sorted(pass_dict.items()): print('PASS: ' + rule + ' [' + key + ']') pass_count = len(pass_dict) # print out the ignored rules - ignore_count, not_used = print_rules(alert_dict, 'IGNORE', config_dict, config_msg, min_level, levels, + ignore_count, not_used = print_rules(zap, alert_dict, 'IGNORE', config_dict, config_msg, min_level, inc_ignore_rules, True, detailed_output, {}) # print out the info rules - info_count, not_used = print_rules(alert_dict, 'INFO', config_dict, config_msg, min_level, levels, + info_count, not_used = print_rules(zap, alert_dict, 'INFO', config_dict, config_msg, min_level, inc_info_rules, info_unspecified, detailed_output, in_progress_issues) # print out the warning rules - warn_count, warn_inprog_count = print_rules(alert_dict, 'WARN', config_dict, config_msg, min_level, levels, + warn_count, warn_inprog_count = print_rules(zap, alert_dict, 'WARN', config_dict, config_msg, min_level, inc_warn_rules, not info_unspecified, detailed_output, in_progress_issues) # print out the failing rules - fail_count, fail_inprog_count = print_rules(alert_dict, 'FAIL', config_dict, config_msg, min_level, levels, + fail_count, fail_inprog_count = print_rules(zap, alert_dict, 'FAIL', config_dict, config_msg, min_level, inc_fail_rules, True, detailed_output, in_progress_issues) if report_html: # Save the report - with open(base_dir + report_html, 'w') as f: - f.write(zap.core.htmlreport()) + write_report(base_dir + report_html, zap.core.htmlreport()) + + if report_json: + # Save the report + write_report(base_dir + report_json, zap.core.jsonreport()) if report_md: # Save the report - with open(base_dir + report_md, 'w') as f: - f.write(zap.core.mdreport()) + write_report(base_dir + report_md, zap.core.mdreport()) if report_xml: # Save the report - with open(base_dir + report_xml, 'w') as f: - f.write(zap.core.xmlreport()) + write_report(base_dir + report_xml, zap.core.xmlreport()) print('FAIL-NEW: ' + str(fail_count) + '\tFAIL-INPROG: ' + str(fail_inprog_count) + '\tWARN-NEW: ' + str(warn_count) + '\tWARN-INPROG: ' + str(warn_inprog_count) + '\tINFO: ' + str(info_count) + '\tIGNORE: ' + str(ignore_count) + '\tPASS: ' + str(pass_count)) + trigger_hook('zap_pre_shutdown', zap) # Stop ZAP zap.core.shutdown() @@ -404,7 +425,7 @@ def main(argv): else: print("ERROR %s" % e) logging.warning('I/O error: ' + str(e)) - dump_log_file(cid) + dump_log_file(cid) except: print("ERROR " + str(sys.exc_info()[0])) @@ -414,9 +435,11 @@ def main(argv): if not running_in_docker(): stop_docker(cid) + trigger_hook('pre_exit', fail_count, warn_count, pass_count) + if fail_count > 0: sys.exit(1) - elif warn_count > 0: + elif (not ignore_warn) and warn_count > 0: sys.exit(2) elif pass_count > 0: sys.exit(0) diff --git a/cicd/jenkins-slave-zap/rhel7/zap/zap-full-scan.py b/cicd/jenkins-slave-zap/rhel7/zap/zap-full-scan.py index b488b51..3435dd8 100755 --- a/cicd/jenkins-slave-zap/rhel7/zap/zap-full-scan.py +++ b/cicd/jenkins-slave-zap/rhel7/zap/zap-full-scan.py @@ -41,7 +41,7 @@ # to be handled differently. # You can also add your own messages for the rules by appending them after a tab # at the end of each line. -# By default all of the active scan rules run but you can prevent rules from +# By default all of the active scan rules run but you can prevent rules from # running by supplying a configuration file with the rules set to IGNORE. import getopt @@ -51,20 +51,18 @@ import os.path import sys import time -import urllib2 from datetime import datetime from zapv2 import ZAPv2 from zap_common import * -timeout = 120 + config_dict = {} config_msg = {} out_of_scope_dict = {} -levels = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] min_level = 0 -# Scan rules that aren't really relevant, eg the examples rules in the alpha set -blacklist = ['-1', '50003', '60000', '60001'] +# Scan rules that aren't really relevant, e.g. the examples rules in the alpha set +blacklist = ['-1', '50003', '60000', '60001', '60100', '60101'] # Scan rules that are being addressed in_progress_issues = {} @@ -76,16 +74,18 @@ def usage(): print('Usage: zap-full-scan.py -t [options]') - print(' -t target target URL including the protocol, eg https://www.example.com') + print(' -t target target URL including the protocol, e.g. https://www.example.com') print('Options:') + print(' -h print this help message') print(' -c config_file config file to use to INFO, IGNORE or FAIL warnings') print(' -u config_url URL of config file to use to INFO, IGNORE or FAIL warnings') print(' -g gen_file generate default config file(all rules set to WARN)') - print(' -m mins the number of minutes to spider for (default 1)') + print(' -m mins the number of minutes to spider for (defaults to no limit)') print(' -r report_html file to write the full ZAP HTML report') print(' -w report_md file to write the full ZAP Wiki(Markdown) report') print(' -x report_xml file to write the full ZAP XML report') - print(' -a include the alpha passive scan rules as well') + print(' -J report_json file to write the full ZAP JSON document') + print(' -a include the alpha active and passive scan rules as well') print(' -d show debug messages') print(' -P specify listen port') print(' -D delay in seconds to wait for passive scanning ') @@ -95,7 +95,9 @@ def usage(): print(' -n context_file context file which will be loaded prior to scanning the target') print(' -p progress_file progress file which specifies issues that are being addressed') print(' -s short output format - dont show PASSes or example URLs') + print(' -T max time in minutes to wait for ZAP to start and the passive scan to run') print(' -z zap_options ZAP command line options e.g. -z "-config aaa=bbb -config ccc=ddd"') + print(' --hook path to python file that define your custom hooks') print('') print('For more details see https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan') @@ -116,6 +118,7 @@ def main(argv): report_html = '' report_md = '' report_xml = '' + report_json = '' target = '' zap_alpha = False info_unspecified = False @@ -124,6 +127,8 @@ def main(argv): zap_ip = 'localhost' zap_options = '' delay = 0 + timeout = 0 + hook_file = None pass_count = 0 warn_count = 0 @@ -134,14 +139,17 @@ def main(argv): fail_inprog_count = 0 try: - opts, args = getopt.getopt(argv, "t:c:u:g:m:n:r:w:x:l:daijp:sz:P:D:") + opts, args = getopt.getopt(argv, "t:c:u:g:m:n:r:J:w:x:l:hdaijp:sz:P:D:T:", ["hook="]) except getopt.GetoptError as exc: logging.warning('Invalid option ' + exc.opt + ' : ' + exc.msg) usage() sys.exit(3) for opt, arg in opts: - if opt == '-t': + if opt == '-h': + usage() + sys.exit(0) + elif opt == '-t': target = arg logging.debug('Target: ' + target) elif opt == '-c': @@ -164,6 +172,8 @@ def main(argv): progress_file = arg elif opt == '-r': report_html = arg + elif opt == '-J': + report_json = arg elif opt == '-w': report_md = arg elif opt == '-x': @@ -176,15 +186,24 @@ def main(argv): ajax = True elif opt == '-l': try: - min_level = levels.index(arg) + min_level = zap_conf_lvls.index(arg) except ValueError: - logging.warning('Level must be one of ' + str(levels)) + logging.warning('Level must be one of ' + str(zap_conf_lvls)) usage() sys.exit(3) elif opt == '-z': zap_options = arg elif opt == '-s': detailed_output = False + elif opt == '-T': + timeout = int(arg) + elif opt == '--hook': + hook_file = arg + + check_zap_client_version() + + load_custom_hooks(hook_file) + trigger_hook('cli_opts', opts) # Check target supplied and ok if len(target) == 0: @@ -198,7 +217,7 @@ def main(argv): if running_in_docker(): base_dir = '/zap/wrk/' - if config_file or generate or report_html or report_xml or progress_file or context_file: + if config_file or generate or report_html or report_xml or report_json or progress_file or context_file: # Check directory has been mounted if not os.path.exists(base_dir): logging.warning('A file based option has been specified but the directory \'/zap/wrk\' is not mounted ') @@ -214,11 +233,19 @@ def main(argv): if config_file: # load config file from filestore with open(base_dir + config_file) as f: - load_config(f, config_dict, config_msg, out_of_scope_dict) + try: + load_config(f, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to load config file " + base_dir + config_file + " " + str(e)) + sys.exit(3) elif config_url: # load config file from url try: - load_config(urllib2.urlopen(config_url), config_dict, config_msg, out_of_scope_dict) + config_data = urlopen(config_url).read().decode('UTF-8').splitlines() + load_config(config_data, config_dict, config_msg, out_of_scope_dict) + except ValueError as e: + logging.warning("Failed to read configs from " + config_url + " " + str(e)) + sys.exit(3) except: logging.warning('Failed to read configs from ' + config_url) sys.exit(3) @@ -238,15 +265,14 @@ def main(argv): params = [ '-config', 'spider.maxDuration=' + str(mins), '-addonupdate', - '-addoninstall', 'pscanrulesBeta'] # In case we're running in the stable container + '-addoninstall', 'pscanrulesBeta', # In case we're running in the stable container + '-addoninstall', 'ascanrulesBeta'] if zap_alpha: - params.append('-addoninstall') - params.append('pscanrulesAlpha') + params.extend(['-addoninstall', 'pscanrulesAlpha']) + params.extend(['-addoninstall', 'ascanrulesAlpha']) - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) start_zap(port, params) @@ -263,14 +289,14 @@ def main(argv): params = [ '-config', 'spider.maxDuration=' + str(mins), '-addonupdate', - '-addoninstall', 'pscanrulesBeta'] # In case we're running in the stable container + '-addoninstall', 'pscanrulesBeta', # In case we're running in the stable container + '-addoninstall', 'ascanrulesBeta'] if (zap_alpha): params.extend(['-addoninstall', 'pscanrulesAlpha']) + params.extend(['-addoninstall', 'ascanrulesAlpha']) - if zap_options: - for zap_opt in zap_options.split(" "): - params.append(zap_opt) + add_zap_options(params, zap_options) try: cid = start_docker_zap('owasp/zap2docker-weekly', port, params, mount_dir) @@ -283,19 +309,14 @@ def main(argv): try: zap = ZAPv2(proxies={'http': 'http://' + zap_ip + ':' + str(port), 'https': 'http://' + zap_ip + ':' + str(port)}) - wait_for_zap_start(zap, timeout) + wait_for_zap_start(zap, timeout * 60) + trigger_hook('zap_started', zap, target) if context_file: # handle the context file, cant use base_dir as it might not have been set up - res = zap.context.import_context('/zap/wrk/' + os.path.basename(context_file)) - if res.startswith("ZAP Error"): - logging.error('Failed to load context file ' + context_file + ' : ' + res) + zap_import_context(zap, '/zap/wrk/' + os.path.basename(context_file)) - # Access the target - res = zap.urlopen(target) - if res.startswith("ZAP Error"): - # errno.EIO is 5, not sure why my atempts to import it failed;) - raise IOError(5, 'Failed to connect') + zap_access_target(zap, target) if target.count('/') > 2: # The url can include a valid path, but always reset to spider the host @@ -331,10 +352,10 @@ def main(argv): zap_active_scan(zap, target, scan_policy) - zap_wait_for_passive_scan(zap) + zap_wait_for_passive_scan(zap, timeout * 60) # Print out a count of the number of urls - num_urls = len(zap.core.urls) + num_urls = len(zap.core.urls()) if num_urls == 0: logging.warning('No URLs found - is the target URL accessible? Local services may not be accessible from the Docker container') else: @@ -383,7 +404,7 @@ def main(argv): if not alert_dict.has_key(plugin_id) and not(config_dict.has_key(plugin_id) and config_dict[plugin_id] == 'IGNORE'): pass_dict[plugin_id] = rule.get('name') - if min_level == levels.index("PASS") and detailed_output: + if min_level == zap_conf_lvls.index("PASS") and detailed_output: for key, rule in sorted(pass_dict.iteritems()): print('PASS: ' + rule + ' [' + key + ']') @@ -399,40 +420,42 @@ def main(argv): print('SKIP: ' + rule.get('name') + ' [' + plugin_id + ']') # print out the ignored rules - ignore_count, not_used = print_rules(alert_dict, 'IGNORE', config_dict, config_msg, min_level, levels, + ignore_count, not_used = print_rules(zap, alert_dict, 'IGNORE', config_dict, config_msg, min_level, inc_ignore_rules, True, detailed_output, {}) # print out the info rules - info_count, not_used = print_rules(alert_dict, 'INFO', config_dict, config_msg, min_level, levels, + info_count, not_used = print_rules(zap, alert_dict, 'INFO', config_dict, config_msg, min_level, inc_info_rules, info_unspecified, detailed_output, in_progress_issues) # print out the warning rules - warn_count, warn_inprog_count = print_rules(alert_dict, 'WARN', config_dict, config_msg, min_level, levels, + warn_count, warn_inprog_count = print_rules(zap, alert_dict, 'WARN', config_dict, config_msg, min_level, inc_warn_rules, not info_unspecified, detailed_output, in_progress_issues) # print out the failing rules - fail_count, fail_inprog_count = print_rules(alert_dict, 'FAIL', config_dict, config_msg, min_level, levels, + fail_count, fail_inprog_count = print_rules(zap, alert_dict, 'FAIL', config_dict, config_msg, min_level, inc_fail_rules, True, detailed_output, in_progress_issues) if report_html: # Save the report - with open(base_dir + report_html, 'w') as f: - f.write(zap.core.htmlreport()) + write_report(base_dir + report_html, zap.core.htmlreport()) + + if report_json: + # Save the report + write_report(base_dir + report_json, zap.core.jsonreport()) if report_md: # Save the report - with open(base_dir + report_md, 'w') as f: - f.write(zap.core.mdreport()) + write_report(base_dir + report_md, zap.core.mdreport()) if report_xml: # Save the report - with open(base_dir + report_xml, 'w') as f: - f.write(zap.core.xmlreport()) + write_report(base_dir + report_xml, zap.core.xmlreport()) print('FAIL-NEW: ' + str(fail_count) + '\tFAIL-INPROG: ' + str(fail_inprog_count) + '\tWARN-NEW: ' + str(warn_count) + '\tWARN-INPROG: ' + str(warn_inprog_count) + '\tINFO: ' + str(info_count) + '\tIGNORE: ' + str(ignore_count) + '\tPASS: ' + str(pass_count)) + trigger_hook('zap_pre_shutdown', zap) # Stop ZAP zap.core.shutdown() @@ -444,7 +467,7 @@ def main(argv): else: print("ERROR %s" % e) logging.warning('I/O error: ' + str(e)) - dump_log_file(cid) + dump_log_file(cid) except: print("ERROR " + str(sys.exc_info()[0])) @@ -454,6 +477,8 @@ def main(argv): if not running_in_docker(): stop_docker(cid) + trigger_hook('pre_exit', fail_count, warn_count, pass_count) + if fail_count > 0: sys.exit(1) elif warn_count > 0: diff --git a/cicd/jenkins-slave-zap/rhel7/zap/zap-webswing.sh b/cicd/jenkins-slave-zap/rhel7/zap/zap-webswing.sh index 49c461d..d873b66 100755 --- a/cicd/jenkins-slave-zap/rhel7/zap/zap-webswing.sh +++ b/cicd/jenkins-slave-zap/rhel7/zap/zap-webswing.sh @@ -1,42 +1,41 @@ #!/bin/sh # # Startup script for the Webswing -# +# # Customised for ZAP running in Docker - just call with no parameters for webswing to start and leave # the docker container running # # Set environment. -export HOME=/zap/webswing-2.3/ -export OPTS="-h 0.0.0.0 -j $HOME/jetty.properties -u $HOME/user.properties -c $HOME/webswing.config" -export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +export HOME=/zap/webswing +export OPTS="-h 0.0.0.0 -j $HOME/jetty.properties -c $HOME/webswing.config" export JAVA_OPTS="-Xmx128M" export LOG=$HOME/webswing.out export PID_PATH_NAME=$HOME/webswing.pid -if [ -z `command -v $0` ]; then +if [ -z `command -v $0` ]; then CURRENTDIR=`pwd` cd `dirname $0` > /dev/null SCRIPTPATH=`pwd`/ cd $CURRENTDIR else - SCRIPTPATH="" + SCRIPTPATH="" fi if [ ! -f $HOME/webswing-server.war ]; then - echo "Webswing executable not found in $HOME folder" + echo "Webswing executable not found in $HOME folder" exit 1 fi if [ ! -f $JAVA_HOME/bin/java ]; then - echo "Java installation not found in $JAVA_HOME folder" + echo "Java installation not found in $JAVA_HOME folder" exit 1 fi if [ -z `command -v xvfb-run` ]; then - echo "Unable to locate xvfb-run command. Please install Xvfb before starting Webswing." + echo "Unable to locate xvfb-run command. Please install Xvfb before starting Webswing." exit 1 fi if [ ! -z `command -v ldconfig` ]; then - if [ `ldconfig -p | grep -i libXext | wc -l` -lt 1 ]; then + if [ `ldconfig -p | grep -i libXext | wc -l` -lt 1 ]; then echo "Missing dependent library libXext." exit 1 fi @@ -65,4 +64,4 @@ case "$1" in xvfb-run $SCRIPTPATH$0 run esac -exit 0 \ No newline at end of file +exit 0 diff --git a/cicd/jenkins-slave-zap/rhel7/zap/zap-x.sh b/cicd/jenkins-slave-zap/rhel7/zap/zap-x.sh index cb1a027..912ae09 100755 --- a/cicd/jenkins-slave-zap/rhel7/zap/zap-x.sh +++ b/cicd/jenkins-slave-zap/rhel7/zap/zap-x.sh @@ -4,4 +4,8 @@ if [ ! -f /tmp/.X1-lock ] then Xvfb :1 -screen 0 1024x768x16 -ac & fi -/zap/zap.sh $@ +/zap/zap.sh "$@" + +# Shutdown xvfb +kill -9 `cat /tmp/.X1-lock` +rm -f /tmp/.X1-lock diff --git a/cicd/jenkins-slave-zap/rhel7/zap/zap_common.py b/cicd/jenkins-slave-zap/rhel7/zap/zap_common.py index f7cb7d8..958fa3c 100755 --- a/cicd/jenkins-slave-zap/rhel7/zap/zap_common.py +++ b/cicd/jenkins-slave-zap/rhel7/zap/zap_common.py @@ -23,15 +23,104 @@ import logging import os import re +import shlex import socket import subprocess import sys import time import traceback import errno +import imp +import zapv2 from random import randint +from six.moves.urllib.request import urlopen +from six import binary_type +try: + import pkg_resources +except ImportError: + # don't hard fail since it's just used for the version check + logging.warning('Error importing pkg_resources. Is setuptools installed?') + +OLD_ZAP_CLIENT_WARNING = '''A newer version of python_owasp_zap_v2.4 + is available. Please run \'pip install -U python_owasp_zap_v2.4\' to update to + the latest version.'''.replace('\n', '') + +zap_conf_lvls = ["PASS", "IGNORE", "INFO", "WARN", "FAIL"] +zap_hooks = None + +def load_custom_hooks(hooks_file=None): + """ Loads a custom python module which modifies zap scripts behaviour + hooks_file - a python file which defines custom hooks + """ + global zap_hooks + hooks_file = hooks_file if hooks_file else os.environ.get('ZAP_HOOKS', '~/.zap_hooks.py') + hooks_file = os.path.expanduser(hooks_file) + + if not os.path.exists(hooks_file): + logging.debug('Could not find custom hooks file at %s ' % hooks_file) + return + + zap_hooks = imp.load_source("zap_hooks", hooks_file) + + +def hook(hook_name=None, **kwargs): + """ + Decorator method for calling hook before/after method. + Always adds a hook that runs before intercepting args and if wrap=True will create + another hook to intercept the return. + hook_name - name of hook for interactions, if None will use the name of the method it wrapped + """ + after_hook = kwargs.get('wrap', False) + def _decorator(func): + name = func.__name__ + _hook_name = hook_name if hook_name else name + def _wrap(*args, **kwargs): + hook_args = list(args) + hook_kwargs = dict(kwargs) + args = trigger_hook(_hook_name, *hook_args, **hook_kwargs) + args_list = list(args) + return_data = func(*args_list, **kwargs) + + if after_hook: + return trigger_hook('%s_wrap' % _hook_name, return_data, **hook_kwargs) + return return_data + return _wrap + return _decorator + + +def trigger_hook(name, *args, **kwargs): + """ Trigger execution of custom hook method if found """ + global zap_hooks + arg_length = len(args) + args_list = list(args) + args = args[0] if arg_length == 1 else args + + logging.debug('Trigger hook: %s, args: %s' % (name, arg_length)) + + if not zap_hooks: + return args + elif not hasattr(zap_hooks, name): + return args + + hook_fn = getattr(zap_hooks, name) + if not callable(hook_fn): + return args + + response = hook_fn(*args_list, **kwargs) + + # The number of args returned should match arguments passed + if not response: + return args + elif arg_length == 1: + return args + elif (isinstance(response, list) or isinstance(response, tuple)) and len(response) != arg_length: + return args + return response + + +@hook() def load_config(config, config_dict, config_msg, out_of_scope_dict): """ Loads the config file specified into: config_dict - a dictionary which maps plugin_ids to levels (IGNORE, WARN, FAIL) @@ -41,11 +130,13 @@ def load_config(config, config_dict, config_msg, out_of_scope_dict): for line in config: if not line.startswith('#') and len(line) > 1: (key, val, optional) = line.rstrip().split('\t', 2) - if key == 'OUTOFSCOPE': - for plugin_id in val.split(','): + if val == 'OUTOFSCOPE': + for plugin_id in key.split(','): if plugin_id not in out_of_scope_dict: out_of_scope_dict[plugin_id] = [] out_of_scope_dict[plugin_id].append(re.compile(optional)) + elif val not in zap_conf_lvls: + raise ValueError("Level {0} is not a supported level: {1}".format(val, zap_conf_lvls)) else: config_dict[key] = val if '\t' in optional: @@ -53,6 +144,7 @@ def load_config(config, config_dict, config_msg, out_of_scope_dict): config_msg[key] = usermsg else: config_msg[key] = '' + logging.debug('Loaded config: {0}'.format(config_dict)) def is_in_scope(plugin_id, url, out_of_scope_dict): @@ -74,7 +166,7 @@ def is_in_scope(plugin_id, url, out_of_scope_dict): return True -def print_rule(action, alert_list, detailed_output, user_msg, in_progress_issues): +def print_rule(zap, action, alert_list, detailed_output, user_msg, in_progress_issues): id = alert_list[0].get('pluginId') if id in in_progress_issues: print (action + '-IN_PROGRESS: ' + alert_list[0].get('alert') + ' [' + id + '] x ' + str(len(alert_list)) + ' ' + user_msg) @@ -83,13 +175,15 @@ def print_rule(action, alert_list, detailed_output, user_msg, in_progress_issues else: print (action + '-NEW: ' + alert_list[0].get('alert') + ' [' + id + '] x ' + str(len(alert_list)) + ' ' + user_msg) if detailed_output: - # Show (up to) first 5 urls + # Show (up to) first 5 urls, along with the response code (which we have to perform another request for) for alert in alert_list[0:5]: - print ('\t' + alert.get('url')) + msg = zap.core.message(alert.get('messageId')) + respHeader = msg['responseHeader'] + code = respHeader[respHeader.index(' ') + 1 : respHeader.index('\r\n')] + print ('\t' + alert.get('url') + ' (' + code + ')') -def print_rules(alert_dict, level, config_dict, config_msg, min_level, levels, inc_rule, inc_extra, detailed_output, in_progress_issues): - # print out the ignored rules +def print_rules(zap, alert_dict, level, config_dict, config_msg, min_level, inc_rule, inc_extra, detailed_output, in_progress_issues): count = 0 inprog_count = 0 for key, alert_list in sorted(alert_dict.items()): @@ -98,13 +192,13 @@ def print_rules(alert_dict, level, config_dict, config_msg, min_level, levels, i user_msg = '' if key in config_msg: user_msg = config_msg[key] - if min_level <= levels.index(level): - print_rule(level, alert_list, detailed_output, user_msg, in_progress_issues) + if min_level <= zap_conf_lvls.index(level): + print_rule(zap, level, alert_list, detailed_output, user_msg, in_progress_issues) if key in in_progress_issues: inprog_count += 1 else: count += 1 - return count, inprog_count + return trigger_hook('print_rules_wrap', count, inprog_count) def inc_ignore_rules(config_dict, key, inc_extra): @@ -149,6 +243,13 @@ def running_in_docker(): return os.path.exists('/.dockerenv') +def add_zap_options(params, zap_options): + if zap_options: + for zap_opt in shlex.split(zap_options): + params.append(zap_opt) + + +@hook() def start_zap(port, extra_zap_params): logging.debug('Starting ZAP') # All of the default common params @@ -160,13 +261,21 @@ def start_zap(port, extra_zap_params): '-config', 'api.addrs.addr.name=.*', '-config', 'api.addrs.addr.regex=true'] + params.extend(extra_zap_params) + + logging.info('Params: ' + str(params)) + with open('zap.out', "w") as outfile: - subprocess.Popen(params + extra_zap_params, stdout=outfile) + subprocess.Popen(params, stdout=outfile) -def wait_for_zap_start(zap, timeout): +def wait_for_zap_start(zap, timeout_in_secs = 600): version = None - for x in range(0, timeout): + if not timeout_in_secs: + # if ZAP doesn't start in 10 mins then its probably not going to start + timeout_in_secs = 600 + + for x in range(0, timeout_in_secs): try: version = zap.core.version logging.debug('ZAP Version ' + version) @@ -178,9 +287,10 @@ def wait_for_zap_start(zap, timeout): if not version: raise IOError( errno.EIO, - 'Failed to connect to ZAP after {0} seconds'.format(timeout)) + 'Failed to connect to ZAP after {0} seconds'.format(timeout_in_secs)) +@hook(wrap=True) def start_docker_zap(docker_image, port, extra_zap_params, mount_dir): try: logging.debug('Pulling ZAP Docker image: ' + docker_image) @@ -222,7 +332,7 @@ def get_free_port(): if not (sock.connect_ex(('127.0.0.1', port)) == 0): return port - + def ipaddress_for_cid(cid): insp_output = subprocess.check_output(['docker', 'inspect', cid]).strip().decode('utf-8') #logging.debug('Docker Inspect: ' + insp_output) @@ -248,6 +358,14 @@ def stop_docker(cid): logging.warning('Docker rm failed') +@hook() +def zap_access_target(zap, target): + res = zap.urlopen(target) + if res.startswith("ZAP Error"): + raise IOError(errno.EIO, 'ZAP failed to access: {0}'.format(target)) + + +@hook(wrap=True) def zap_spider(zap, target): logging.debug('Spider ' + target) spider_scan_id = zap.spider.scan(target) @@ -259,6 +377,7 @@ def zap_spider(zap, target): logging.debug('Spider complete') +@hook(wrap=True) def zap_ajax_spider(zap, target, max_time): logging.debug('AjaxSpider ' + target) if max_time: @@ -272,6 +391,7 @@ def zap_ajax_spider(zap, target, max_time): logging.debug('Ajax Spider complete') +@hook(wrap=True) def zap_active_scan(zap, target, policy): logging.debug('Active Scan ' + target + ' with policy ' + policy) ascan_scan_id = zap.ascan.scan(target, recurse=True, scanpolicyname=policy) @@ -284,15 +404,25 @@ def zap_active_scan(zap, target, policy): logging.debug(zap.ascan.scan_progress(ascan_scan_id)) -def zap_wait_for_passive_scan(zap): +def zap_wait_for_passive_scan(zap, timeout_in_secs = 0): rtc = zap.pscan.records_to_scan logging.debug('Records to scan...') + time_taken = 0 + timed_out = False while (int(zap.pscan.records_to_scan) > 0): logging.debug('Records to passive scan : ' + zap.pscan.records_to_scan) time.sleep(2) - logging.debug('Passive scanning complete') + time_taken += 2 + if timeout_in_secs and time_taken > timeout_in_secs: + timed_out = True + break + if timed_out: + logging.debug('Exceeded passive scan timeout') + else: + logging.debug('Passive scanning complete') +@hook(wrap=True) def zap_get_alerts(zap, baseurl, blacklist, out_of_scope_dict): # Retrieve the alerts using paging in case there are lots of them st = 0 @@ -319,3 +449,67 @@ def zap_get_alerts(zap, baseurl, blacklist, out_of_scope_dict): alerts = zap.core.alerts(start=st, count=pg) logging.debug('Total number of alerts: ' + str(alert_count)) return alert_dict + + +def get_latest_zap_client_version(): + version_info = None + + try: + version_info = urlopen('https://pypi.python.org/pypi/python-owasp-zap-v2.4/json', timeout=10) + except Exception as e: + logging.warning('Error fetching latest ZAP Python API client version: %s' % e) + return None + + if version_info is None: + logging.warning('Error fetching latest ZAP Python API client version: %s' % e) + return None + + version_json = json.loads(version_info.read().decode('utf-8')) + + if 'info' not in version_json: + logging.warning('No version found for latest ZAP Python API client.') + return None + if 'version' not in version_json['info']: + logging.warning('No version found for latest ZAP Python API client.') + return None + + return version_json['info']['version'] + + +def check_zap_client_version(): + # No need to check ZAP Python API client while running in Docker + if running_in_docker(): + return + + if 'pkg_resources' not in globals(): # import failed + logging.warning('Could not check ZAP Python API client without pkg_resources.') + return + + current_version = getattr(zapv2, '__version__', None) + latest_version = get_latest_zap_client_version() + parse_version = pkg_resources.parse_version + if current_version and latest_version and \ + parse_version(current_version) < parse_version(latest_version): + logging.warning(OLD_ZAP_CLIENT_WARNING) + elif current_version is None: + # the latest versions >= 0.0.9 have a __version__ + logging.warning(OLD_ZAP_CLIENT_WARNING) + # else: + # we're up to date or ahead / running a development build + # or latest_version is None and the user already got a warning + + +def write_report(file_path, report): + with open(file_path, mode='wb') as f: + if not isinstance(report, binary_type): + report = report.encode('utf-8') + + f.write(report) + +@hook(wrap=True) +def zap_import_context(zap, context_file): + res = context_id = zap.context.import_context(context_file) + if res.startswith("ZAP Error"): + context_id = None + logging.error('Failed to load context file ' + context_file + ' : ' + res) + return context_id