From e59d07fe41e73394cd97b89a2f6b24e13be1c25b Mon Sep 17 00:00:00 2001 From: Taba Imre Date: Mon, 21 Jan 2019 10:23:47 +0100 Subject: [PATCH 1/7] Testing access --- test.asd | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.asd diff --git a/test.asd b/test.asd new file mode 100644 index 0000000..f47d76e --- /dev/null +++ b/test.asd @@ -0,0 +1 @@ +sddasd From 68af4ef280f2f436e44f05bc5d73821396048a86 Mon Sep 17 00:00:00 2001 From: Taba Imre Date: Mon, 21 Jan 2019 10:26:55 +0100 Subject: [PATCH 2/7] Adding image entropy calculator Ugly but works phase Changes have been made Fixing failed commit Adding the whole thing into _make_http_request Removed unnecesarry global variable Remove debug message Added CHANGES.md Update CHANGES.md --- CHANGES.md | 28 +++++++++++++++++++++++++++ UploadScanner.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++- test.asd | 1 - 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 CHANGES.md delete mode 100644 test.asd diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..8a3a728 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,28 @@ +## Main feature + +This module was created to roughly calculate randomness in redownloaded pictures. + +It can be used to detect possible memory leaks from image renderers. +If there is some error in the image renderer logic, and it is not handling exceptions properly, +it is possible that a malformed uploaded file will trigger it, and the response will contain parts of the +image renderer memory. +This behaviour will increase the randomness of pixels in the returned picture. + +By getting the full RGB list from the response picture, and compressing it with zlib, we can +estimate the randomness of the pixels by dividing these two values with each other. + +This module will give an informational issue to all redownloaded pictures, with the compression ratio, +which can help us to see, if there is too much random noise in the picture, and from then, we can manually +investigate the incident. + + +## Changes in the code: + +Added a checkbox to the general options UI with label "Calculate image entropy". It's referenced in the code as calculate_entropy + +Added a new function called _calculate_image_entropy, which is defined at line 866. +This function takes a redownloaded response, and tries to get RGB list out of its body. +Then it compresses the RGB list and calculates a ratio based on the two results. + +This function is called from the _make_http_request function after a redownloaded response returns with an image content type + and the calculate_entropy checkbox is checked. The call itself can be seen at line 4427. diff --git a/UploadScanner.py b/UploadScanner.py index 95b272d..a62a0fd 100755 --- a/UploadScanner.py +++ b/UploadScanner.py @@ -862,6 +862,39 @@ def add_log_entry(self, rr): self._helpers.analyzeRequest(rr).getUrl())) self.fireTableRowsInserted(row, row) + # Function to calculate image entropy via compressing redownloaded images + def _calculate_image_entropy(self, injector, urr): + + iResponseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) + body = FloydsHelpers.jb2ps(urr.download_rr.getResponse())[iResponseInfo.getBodyOffset():] + + service = urr.download_rr.getHttpService() + url = self._helpers.analyzeRequest(urr.download_rr).getUrl() + + try: + rgbs = ImageHelpers.get_image_rgb_list(body) + conv = "".join([struct.pack("
" \ + "RGB numbers before compression: {}
" \ + "RGB numbers after compression: {}
" \ + "Compression ratio: {}
".format(len(rgbs)*4, len(compressed), ratio) + + csi = CustomScanIssue([injector.get_brr()], name, detail, confidence, severity, service, url) + csi.httpMessagesPy = [urr.upload_rr, urr.download_rr] + # Using direct call to burp callback, to reduce noise in the output logs. + self._callbacks.addScanIssue(csi) + + except: + print "Could not get image entropy." + + # Implement IHttpListener def processHttpMessage(self, _, messageIsRequest, base_request_response): try: @@ -890,6 +923,7 @@ def processHttpMessage(self, _, messageIsRequest, base_request_response): # ... do not scan things that are not "in scope" (see DownloadMatcherCollection class) # means we only check if we uploaded stuff to that host or the user configured # another host in the ReDownloader options that is therefore also "in scope" + matchers = self.dl_matchers.get_matchers_for_url(url) if not matchers: #We hit this for all not "in scope" requests @@ -4388,6 +4422,14 @@ def _make_http_request(self, injector, req, report_timeouts=True, throttle=True, preflight_rr, download_rr = injector.opts.redownloader_try_redownload(resp, redownload_filename) urr.preflight_rr = preflight_rr urr.download_rr = download_rr + + # Calculating image entropy + if injector.opts.calculate_entropy: + download_responseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) + headers = [FloydsHelpers.u2s(x) for x in download_responseInfo.getHeaders()] + if any("image/" in h for h in headers): + self._calculate_image_entropy(injector, urr) + if injector.opts.create_log: # create a new log entry with the message details if urr.preflight_rr: @@ -8418,6 +8460,7 @@ def __init__(self, burp_extender, callbacks, helpers, scan_controler=None, globa # Options general: self.throttle_time = 0.0 self.sleep_time = 6.0 + self.calculate_entropy = False self.create_log = False self.replace_filename = True self.replace_ct = True @@ -8497,6 +8540,7 @@ def serialize(self): serialized_object['throttle_time'] = self.throttle_time serialized_object['sleep_time'] = self.sleep_time + serialized_object['calculate_entropy'] = self.calculate_entropy serialized_object['create_log'] = self.create_log serialized_object['replace_filename'] = self.replace_filename serialized_object['replace_ct'] = self.replace_ct @@ -8565,6 +8609,7 @@ def deserialize(self, serialized_object, global_to_tab=False): # This "if" is necessary to be backward compatible (the old serialized object does not have this attribute) if 'sleep_time' in serialized_object: self.tf_sleep_time.setText(str(serialized_object['sleep_time'])) + self.cb_calculate_entropy.setSelected(serialized_object['calculate_entropy']) self.cb_create_log.setSelected(serialized_object['create_log']) self.cb_replace_filename.setSelected(serialized_object['replace_filename']) self.cb_replace_ct.setSelected(serialized_object['replace_ct']) @@ -8764,6 +8809,7 @@ def create_options(self): text=self.image_exiftool) self.lbl_throttle_time, self.tf_throttle_time = self.small_tf("Throttle between requests in seconds:", str(self.throttle_time)) self.lbl_sleep_time, self.tf_sleep_time = self.small_tf("Sleep time for sleep payloads in seconds:", str(self.sleep_time)) + _, self.cb_calculate_entropy = self.checkbox('Calculate entropy for images:', self.calculate_entropy) _, self.cb_create_log = self.checkbox('Create log, see "Done uploads" tab:', self.create_log) _, self.cb_replace_filename = self.checkbox('Replace filename in requests:', self.replace_filename) _, self.cb_replace_ct = self.checkbox('Replace content type in requests:', self.replace_ct) @@ -8906,7 +8952,8 @@ def insertUpdate(self, _): except ValueError: self.sleep_time = 6.0 OptionsPanel.mark_misconfigured(self.lbl_sleep_time) - + + self.calculate_entropy = self.cb_calculate_entropy.isSelected() self.create_log = self.cb_create_log.isSelected() self.replace_filename = self.cb_replace_filename.isSelected() self.replace_ct = self.cb_replace_ct.isSelected() diff --git a/test.asd b/test.asd deleted file mode 100644 index f47d76e..0000000 --- a/test.asd +++ /dev/null @@ -1 +0,0 @@ -sddasd From 04b3bd19661ad4ece6b066f335adaf27ca402878 Mon Sep 17 00:00:00 2001 From: Taba Imre Date: Mon, 4 Feb 2019 13:50:53 +0100 Subject: [PATCH 3/7] Reporting after all the modules/median calculation --- CHANGES.md | 18 ++++++++++++++---- UploadScanner.py | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8a3a728..d20c91b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,18 +11,28 @@ This behaviour will increase the randomness of pixels in the returned picture. By getting the full RGB list from the response picture, and compressing it with zlib, we can estimate the randomness of the pixels by dividing these two values with each other. -This module will give an informational issue to all redownloaded pictures, with the compression ratio, -which can help us to see, if there is too much random noise in the picture, and from then, we can manually -investigate the incident. +This module will give an informational issue to all redownloaded pictures, where the compression ratio is beneath the +median of all compression ratios, which can help us to see, if there is too much random noise in the picture, +and from then, we can manually investigate the incident. ## Changes in the code: Added a checkbox to the general options UI with label "Calculate image entropy". It's referenced in the code as calculate_entropy +Added a global variable called IMAGE_ENTROPY_CSI (line 155) to store the issues and ratios of the redownloaded pictures. + Added a new function called _calculate_image_entropy, which is defined at line 866. This function takes a redownloaded response, and tries to get RGB list out of its body. Then it compresses the RGB list and calculates a ratio based on the two results. This function is called from the _make_http_request function after a redownloaded response returns with an image content type - and the calculate_entropy checkbox is checked. The call itself can be seen at line 4427. +and the calculate_entropy checkbox is checked. The call itself can be seen at line 4456. + +Added a new helper function which calculates the median of a given list. (line 4182) + +The reporting itself is done after all the other modules are finished their testing, but before the DoS module. +It can be seen at line 1229, where the median of the ratios are calculated, and all the pictures, which are beneath +the median are reported as an informal issue. + +Please note that this code is only a Proof of Concept, the methodology, or places of the functions/calls could be changed after reconsideration. diff --git a/UploadScanner.py b/UploadScanner.py index a62a0fd..219a7c7 100755 --- a/UploadScanner.py +++ b/UploadScanner.py @@ -152,6 +152,7 @@ class BurpExtender(IBurpExtender, IScannerCheck, REDL_FILENAME_MARKER = "${FILENAME}" PYTHON_STR_MARKER_START = "${PYTHONSTR:" PYTHON_STR_MARKER_END = "}" + IMAGE_ENTROPY_CSI = [] # Implement IBurpExtender def registerExtenderCallbacks(self, callbacks): @@ -888,11 +889,10 @@ def _calculate_image_entropy(self, injector, urr): csi = CustomScanIssue([injector.get_brr()], name, detail, confidence, severity, service, url) csi.httpMessagesPy = [urr.upload_rr, urr.download_rr] - # Using direct call to burp callback, to reduce noise in the output logs. - self._callbacks.addScanIssue(csi) + return csi, ratio except: - print "Could not get image entropy." + return None, None # Implement IHttpListener @@ -1226,6 +1226,20 @@ def do_checks(self, injector): # Just to make sure (maybe we write a new module above and forget this call): self.collab_monitor_thread.add_or_update(burp_colab, colab_tests) + # Calculating image entropy is done after all the scans are compleated + if not scan_was_stopped and injector.opts.calculate_entropy: + ratios = [] + # Calculates the median of the compression ratios + for csi, ratio in self.IMAGE_ENTROPY_CSI: + ratios.append(ratio) + median = self._median(ratios) + + print "Reporting Image entropies" + for csi, ratio in self.IMAGE_ENTROPY_CSI: + if ratio < median: + self._callbacks.addScanIssue(csi) + self.IMAGE_ENTROPY_CSI = [] + # DoSing the server is best done at the end when we already know about everything else... # Timeout and DoS - generic if not scan_was_stopped: @@ -4164,6 +4178,17 @@ def _timeout_and_dos(self, injector): csi = self._create_issue_template(brr, title, desc, "Tentative", "Medium") self._add_scan_issue(csi) + + # Helper function to calculate median + def _median(self, lst): + n = len(lst) + if n < 1: + return None + if n % 2 == 1: + return sorted(lst)[n//2] + else: + return sum(sorted(lst)[n//2-1:n//2+1])/2.0 + # Helper functions def _filename_to_expected(self, filename): # TODO feature: maybe try to download both? @@ -4428,7 +4453,9 @@ def _make_http_request(self, injector, req, report_timeouts=True, throttle=True, download_responseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) headers = [FloydsHelpers.u2s(x) for x in download_responseInfo.getHeaders()] if any("image/" in h for h in headers): - self._calculate_image_entropy(injector, urr) + csi, ratio = self._calculate_image_entropy(injector, urr) + if csi is not None: + self.IMAGE_ENTROPY_CSI.append([csi, ratio]) if injector.opts.create_log: # create a new log entry with the message details From ab68a3b388e03b7311a3b1b316b13e28e59d1047 Mon Sep 17 00:00:00 2001 From: b Date: Tue, 23 Apr 2019 11:15:17 +0200 Subject: [PATCH 4/7] Stability fixes --- UploadScanner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/UploadScanner.py b/UploadScanner.py index 219a7c7..223bf76 100755 --- a/UploadScanner.py +++ b/UploadScanner.py @@ -4449,7 +4449,7 @@ def _make_http_request(self, injector, req, report_timeouts=True, throttle=True, urr.download_rr = download_rr # Calculating image entropy - if injector.opts.calculate_entropy: + if injector.opts.calculate_entropy and urr.download_rr is not None: download_responseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) headers = [FloydsHelpers.u2s(x) for x in download_responseInfo.getHeaders()] if any("image/" in h for h in headers): @@ -8636,7 +8636,8 @@ def deserialize(self, serialized_object, global_to_tab=False): # This "if" is necessary to be backward compatible (the old serialized object does not have this attribute) if 'sleep_time' in serialized_object: self.tf_sleep_time.setText(str(serialized_object['sleep_time'])) - self.cb_calculate_entropy.setSelected(serialized_object['calculate_entropy']) + if 'calculate_entropy' in serialized_object: + self.cb_calculate_entropy.setSelected(serialized_object['calculate_entropy']) self.cb_create_log.setSelected(serialized_object['create_log']) self.cb_replace_filename.setSelected(serialized_object['replace_filename']) self.cb_replace_ct.setSelected(serialized_object['replace_ct']) From cd333e47baf026e793db4f675e8376e3435a39e3 Mon Sep 17 00:00:00 2001 From: Taba Imre Date: Mon, 29 Apr 2019 12:31:36 +0200 Subject: [PATCH 5/7] Added for fp, fuzzer, recursive uploader --- UploadScanner.py | 78 +++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/UploadScanner.py b/UploadScanner.py index 223bf76..a6af206 100755 --- a/UploadScanner.py +++ b/UploadScanner.py @@ -152,7 +152,6 @@ class BurpExtender(IBurpExtender, IScannerCheck, REDL_FILENAME_MARKER = "${FILENAME}" PYTHON_STR_MARKER_START = "${PYTHONSTR:" PYTHON_STR_MARKER_END = "}" - IMAGE_ENTROPY_CSI = [] # Implement IBurpExtender def registerExtenderCallbacks(self, callbacks): @@ -873,8 +872,19 @@ def _calculate_image_entropy(self, injector, urr): url = self._helpers.analyzeRequest(urr.download_rr).getUrl() try: + picture_width, picture_height, fileformat = ImageHelpers.image_width_height(body) + if picture_width and picture_height and fileformat: + if picture_width >= 200 or picture_height >= 200: + # We first resize the picture to a small one so we don't have to check + # too many pixels (performance)... + thumbnail = ImageHelpers.rescale_image(200, 200, body) + if thumbnail: # only if body was an image that ImageIO can parse + # Now get the pixels of the picture + body = thumbnail + rgbs = ImageHelpers.get_image_rgb_list(body) conv = "".join([struct.pack(" Date: Tue, 30 Apr 2019 15:36:26 +0200 Subject: [PATCH 6/7] Separate reporting/calculation --- UploadScanner.py | 145 ++++++++++++++++++++++++++++------------------- 1 file changed, 87 insertions(+), 58 deletions(-) diff --git a/UploadScanner.py b/UploadScanner.py index a6af206..c5f09f9 100755 --- a/UploadScanner.py +++ b/UploadScanner.py @@ -862,47 +862,64 @@ def add_log_entry(self, rr): self._helpers.analyzeRequest(rr).getUrl())) self.fireTableRowsInserted(row, row) + # Helper functions for image entropy calculation + def _calculate_image_entropy_wrapper(self, injector, urr): + download_responseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) + headers = [FloydsHelpers.u2s(x) for x in download_responseInfo.getHeaders()] + if any("image/" in h for h in headers): + ratio = self._calculate_image_entropy(injector, urr) + return ratio + + def _report_image_entropy(self, Entropy_list): + # Calculate median of the ratios + ratios = [ratio[0] for ratio in Entropy_list] + median = self._median(ratios) + for item in Entropy_list: + if item[0] and item[0] < median: + injector = item[1] + urr = item[2] + ratio = item[0] + service = urr.download_rr.getHttpService() + url = self._helpers.analyzeRequest(urr.download_rr).getUrl() + name = "Downloaded image entropy info" + severity = "Information" + confidence = "Firm" + detail = "Information about the redownloaded image entropy where compression ratios were lower than the avarage.
" \ + "If the compression ratio is too low, that could mean that the image contains memory leak.

" \ + "The avarage compression ratio for the module was: {}

" \ + "Compression ratio for this image: {}
".format(median,ratio) + + csi = CustomScanIssue([injector.get_brr()], name, detail, confidence, severity, service, url) + csi.httpMessagesPy = [urr.upload_rr, urr.download_rr] + if csi: + self._callbacks.addScanIssue(csi) + + + # Function to calculate image entropy via compressing redownloaded images def _calculate_image_entropy(self, injector, urr): - - iResponseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) - body = FloydsHelpers.jb2ps(urr.download_rr.getResponse())[iResponseInfo.getBodyOffset():] - - service = urr.download_rr.getHttpService() - url = self._helpers.analyzeRequest(urr.download_rr).getUrl() + download_iResponseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) + download_body = FloydsHelpers.jb2ps(urr.download_rr.getResponse())[download_iResponseInfo.getBodyOffset():] try: - picture_width, picture_height, fileformat = ImageHelpers.image_width_height(body) + # Rescaling the image for performance + picture_width, picture_height, fileformat = ImageHelpers.image_width_height(download_body) if picture_width and picture_height and fileformat: if picture_width >= 200 or picture_height >= 200: - # We first resize the picture to a small one so we don't have to check - # too many pixels (performance)... - thumbnail = ImageHelpers.rescale_image(200, 200, body) - if thumbnail: # only if body was an image that ImageIO can parse - # Now get the pixels of the picture - body = thumbnail - - rgbs = ImageHelpers.get_image_rgb_list(body) - conv = "".join([struct.pack("
" \ - "RGB numbers before compression: {}
" \ - "RGB numbers after compression: {}
" \ - "Compression ratio: {}
".format(len(rgbs)*4, len(compressed), ratio) - - csi = CustomScanIssue([injector.get_brr()], name, detail, confidence, severity, service, url) - csi.httpMessagesPy = [urr.upload_rr, urr.download_rr] - return csi, ratio + ratio = len(download_rgbs)*4/(len(download_compressed)*1.0) + return ratio + except: - return None, None + return None # Implement IHttpListener @@ -3387,6 +3404,7 @@ def _polyglot(self, injector, burp_colab): return colab_tests def _fingerping(self, injector): + Entropy_list = [] if not injector.opts.file_formats['png'].isSelected(): # we only upload PNG files in this module return @@ -3418,12 +3436,10 @@ def _fingerping(self, injector): body = resp[body_offset:] # Calculate image entropy - if injector.opts.calculate_entropy and urr.download_rr: - headers = [FloydsHelpers.u2s(x) for x in i_response_info.getHeaders()] - if any("image/" in h for h in headers): - csi, ratio = self._calculate_image_entropy(injector, urr) - if csi is not None: - self._callbacks.addScanIssue(csi) + if injector.opts.calculate_entropy and urr and urr.download_rr: + ratio = self._calculate_image_entropy_wrapper(injector, urr) + if ratio: + Entropy_list.append([ratio, injector, urr]) if body.startswith('\x89PNG'): # print "Downloaded", orig_filename, "is a PNG. Content:" @@ -3485,6 +3501,12 @@ def _fingerping(self, injector): issue = self._create_issue_template(injector.get_brr(), title, desc, confidence, "Information") self._add_scan_issue(issue) + # Reportin image entropy results + if injector.opts.calculate_entropy: + print "Reporting image entropies for fingerping..." + self._report_image_entropy(Entropy_list) + + def _quirks_with_passive(self, injector): if not injector.get_uploaded_filename(): @@ -3684,6 +3706,10 @@ def _recursive_upload_files(self, injector, burp_colab): # file extension: from original request, from file, using the default, guessing by mime type # mime_type: from original request, guessing by file, guessing by file extension # File content: Always from file + + # List for entropy calc + Entropy_list = [] + if not injector.opts.ru_dirpath: return @@ -3739,13 +3765,10 @@ def _recursive_upload_files(self, injector, burp_colab): urr = self._make_http_request(injector, req, redownload_filename=new_filename) # Image entropy calculation - if injector.opts.calculate_entropy and urr.download_rr: - download_responseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) - headers = [FloydsHelpers.u2s(x) for x in download_responseInfo.getHeaders()] - if any("image/" in h for h in headers): - csi, ratio = self._calculate_image_entropy(injector, urr) - if csi is not None: - self._callbacks.addScanIssue(csi) + if injector.opts.calculate_entropy and urr and urr.download_rr: + ratio = self._calculate_image_entropy_wrapper(injector, urr) + if ratio: + Entropy_list.append([ratio, injector, urr]) # Combine with replacer if injector.opts.ru_combine_with_replacer and burp_colab: @@ -3761,9 +3784,17 @@ def _recursive_upload_files(self, injector, burp_colab): urr = self._make_http_request(injector, req) if urr: colab_tests.append(ColabTest(colab_url, urr, issue)) + + if injector.opts.calculate_entropy: + print "Reporting image entropies for recursive uploader..." + self._report_image_entropy(Entropy_list) + return colab_tests def _fuzz(self, injector): + # Variable for image entropy calc + Entropy_list = [] + content = injector.get_uploaded_content() if not content: return @@ -3781,13 +3812,10 @@ def _fuzz(self, injector): if req: urr = self._make_http_request(injector, req, redownload_filename=new_filename) # Image entropy calculation - if injector.opts.calculate_entropy and urr.download_rr: - download_responseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) - headers = [FloydsHelpers.u2s(x) for x in download_responseInfo.getHeaders()] - if any("image/" in h for h in headers): - csi, ratio = self._calculate_image_entropy(injector, urr) - if csi is not None: - self._callbacks.addScanIssue(csi) + if injector.opts.calculate_entropy and urr and urr.download_rr: + ratio = self._calculate_image_entropy_wrapper(injector, urr) + if ratio: + Entropy_list.append([ratio, injector, urr]) for _ in xrange(0, injector.opts.fuzzer_random_mutations): new_content = copy.copy(content) @@ -3809,13 +3837,14 @@ def _fuzz(self, injector): if req: urr = self._make_http_request(injector, req, redownload_filename=new_filename) # Image entropy calculation - if injector.opts.calculate_entropy and urr.download_rr: - download_responseInfo = self._helpers.analyzeResponse(urr.download_rr.getResponse()) - headers = [FloydsHelpers.u2s(x) for x in download_responseInfo.getHeaders()] - if any("image/" in h for h in headers): - csi, ratio = self._calculate_image_entropy(injector, urr) - if csi is not None: - self._callbacks.addScanIssue(csi) + if injector.opts.calculate_entropy and urr and urr.download_rr: + ratio = self._calculate_image_entropy_wrapper(injector, urr) + if ratio: + Entropy_list.append([ratio, injector, urr]) + # Reporting image entropy + if injector.opts.calculate_entropy: + print "Reporting image entropies for fuzzer..." + self._report_image_entropy(Entropy_list) def _timeout_and_dos(self, injector): orig_filename = injector.get_uploaded_filename() From 080a47e3c845b9207c75275e5fd55683788cf3d6 Mon Sep 17 00:00:00 2001 From: Taba Imre Date: Thu, 2 May 2019 11:41:16 +0200 Subject: [PATCH 7/7] Update CHANGES.md --- CHANGES.md | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d20c91b..53ae376 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,26 +13,19 @@ estimate the randomness of the pixels by dividing these two values with each oth This module will give an informational issue to all redownloaded pictures, where the compression ratio is beneath the median of all compression ratios, which can help us to see, if there is too much random noise in the picture, -and from then, we can manually investigate the incident. +and from there, we can manually investigate the incident. +## When/How it is used + +Image entropy calculation is only done on the Fingerping, fuzzer, and recursive uploader module, if the "Calculate image entropy" checkbox is selected. +The module will create informational issues where the redownloaded picture compression ratio is lower than the avarage of it. It can be used to detect issues such as this: https://blog.silentsignal.eu/2019/04/18/drop-by-drop-bleeding-through-libvips/ ## Changes in the code: Added a checkbox to the general options UI with label "Calculate image entropy". It's referenced in the code as calculate_entropy -Added a global variable called IMAGE_ENTROPY_CSI (line 155) to store the issues and ratios of the redownloaded pictures. - -Added a new function called _calculate_image_entropy, which is defined at line 866. -This function takes a redownloaded response, and tries to get RGB list out of its body. -Then it compresses the RGB list and calculates a ratio based on the two results. - -This function is called from the _make_http_request function after a redownloaded response returns with an image content type -and the calculate_entropy checkbox is checked. The call itself can be seen at line 4456. - -Added a new helper function which calculates the median of a given list. (line 4182) +Added new functions for the entropy calculation/reporting. These are defined from line 866 to 922. -The reporting itself is done after all the other modules are finished their testing, but before the DoS module. -It can be seen at line 1229, where the median of the ratios are calculated, and all the pictures, which are beneath -the median are reported as an informal issue. +The calculation is called in the affected modules(Fingerping/Fuzzer/Recursive uploader), where all the ratios and request/responses are stored in a local variable called Entropy_list, which is used at the end of each module for the reporting. -Please note that this code is only a Proof of Concept, the methodology, or places of the functions/calls could be changed after reconsideration. +Added a new helper function which calculates the median of a given list. (line 4243)