Skip to content

Commit 00f03c4

Browse files
author
Simon Planzer
authored
Clear cache (#39)
* cache clean up
1 parent ef71e3a commit 00f03c4

6 files changed

Lines changed: 220 additions & 48 deletions

File tree

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ services:
33

44
branches:
55
only:
6-
- crs_handling
6+
- clear_cache
77

88
env:
99
global:

‎linz-data-importer/gui/Service_dialog_base.ui‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@
1313
<height>682</height>
1414
</rect>
1515
</property>
16+
<property name="focusPolicy">
17+
<enum>Qt::ClickFocus</enum>
18+
</property>
1619
<property name="windowTitle">
1720
<string>LINZ Data Importer</string>
1821
</property>
@@ -801,6 +804,57 @@
801804
</item>
802805
</layout>
803806
</widget>
807+
<tabstops>
808+
<tabstop>uTextFilter</tabstop>
809+
<tabstop>uCRSCombo</tabstop>
810+
<tabstop>uBtnImport</tabstop>
811+
<tabstop>uListOptions</tabstop>
812+
<tabstop>uTableView</tabstop>
813+
<tabstop>uTextDescription</tabstop>
814+
<tabstop>uComboBoxDomain</tabstop>
815+
<tabstop>uBtnAddDomain</tabstop>
816+
<tabstop>uTextAPIKey5</tabstop>
817+
<tabstop>uTextAPIKey1</tabstop>
818+
<tabstop>uTextAPIKey2</tabstop>
819+
<tabstop>uBtnSaveDomain7</tabstop>
820+
<tabstop>uTextAPIKey4</tabstop>
821+
<tabstop>uTextAPIKey8</tabstop>
822+
<tabstop>uBtnSaveDomain1</tabstop>
823+
<tabstop>uTextAPIKey6</tabstop>
824+
<tabstop>uTextAPIKey10</tabstop>
825+
<tabstop>uTextAPIKey9</tabstop>
826+
<tabstop>uTextDomain1</tabstop>
827+
<tabstop>uTextDomain10</tabstop>
828+
<tabstop>uTextDomain8</tabstop>
829+
<tabstop>uTextDomain9</tabstop>
830+
<tabstop>uBtnSaveDomain6</tabstop>
831+
<tabstop>uBtnSaveDomain3</tabstop>
832+
<tabstop>uBtnSaveDomain10</tabstop>
833+
<tabstop>uBtnSaveDomain8</tabstop>
834+
<tabstop>uBtnSaveDomain2</tabstop>
835+
<tabstop>uTextDomain6</tabstop>
836+
<tabstop>uTextDomain7</tabstop>
837+
<tabstop>uTextDomain4</tabstop>
838+
<tabstop>uTextDomain5</tabstop>
839+
<tabstop>uBtnRemoveDomain1</tabstop>
840+
<tabstop>uTextAPIKey3</tabstop>
841+
<tabstop>uBtnSaveDomain4</tabstop>
842+
<tabstop>uBtnSaveDomain9</tabstop>
843+
<tabstop>uTextDomain2</tabstop>
844+
<tabstop>uTextDomain3</tabstop>
845+
<tabstop>uBtnSaveDomain5</tabstop>
846+
<tabstop>uTextAPIKey7</tabstop>
847+
<tabstop>uBtnRemoveDomain2</tabstop>
848+
<tabstop>uBtnRemoveDomain3</tabstop>
849+
<tabstop>uBtnRemoveDomain4</tabstop>
850+
<tabstop>uBtnRemoveDomain5</tabstop>
851+
<tabstop>uBtnRemoveDomain6</tabstop>
852+
<tabstop>uBtnRemoveDomain7</tabstop>
853+
<tabstop>uBtnRemoveDomain8</tabstop>
854+
<tabstop>uBtnRemoveDomain9</tabstop>
855+
<tabstop>uBtnRemoveDomain10</tabstop>
856+
<tabstop>hAboutHtml</tabstop>
857+
</tabstops>
804858
<resources/>
805859
<connections/>
806860
</ui>

‎linz-data-importer/linz_data_importer.py‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@
1515
***************************************************************************/
1616
"""
1717

18-
# This program is released under the terms of the 3 clause BSD license. See the
19-
# LICENSE file for more information.
2018
import sip
2119
sip.setapi('QString', 2)
2220

@@ -32,11 +30,11 @@
3230
from service_data import ServiceData, Localstore, ApiKey
3331

3432
import re
35-
import urllib.request
3633
import threading
3734
import time
3835
import os.path
3936
from owslib import wfs, wms, wmts
37+
from urllib2 import urlopen
4038

4139
# Initialize Qt resources from file resources.py
4240
import resources
@@ -432,20 +430,23 @@ def run(self):
432430
self.updateServiceDataCache()
433431
self.dlg.show()
434432

433+
def purgeCache(self):
434+
"""
435+
Delete any cache files that are not the most current
436+
"""
437+
438+
self.local_store.purgeCache()
439+
435440
def updateServiceDataCache(self):
436441
"""
437442
Update the local cache by deleting the locally stored capability
438443
documents and then re-fetching from the associated web resource
439444
"""
440445

441-
while not self.services_loaded:
442-
time.sleep(10)
443-
continue
444446
self.services_loaded=False
445-
self.local_store.delAllLocalServiceXML()
446-
t=threading.Thread(target=self.loadAllServices)
447+
t = threading.Thread(target=self.loadAllServices, args=(True,))
447448
t.start()
448-
self.cache_updated=True #Needs to employ observer
449+
self.cache_updated=True
449450

450451
def loadUi(self):
451452
"""
@@ -460,21 +461,22 @@ def loadUi(self):
460461
else:
461462
self.dlg.uLabelWarning.hide()
462463

463-
def loadAllServices(self):
464+
def loadAllServices(self, update_cache=False):
464465
"""
465466
Iterate over all domains and service types (WMS, WMTS, WFS).
466467
Request, process, store and format capability documents
467468
"""
468469

469-
all_data=[]
470+
all_data=[]
470471
for domain in self.api_key_instance.getApiKeys():
471472
for service in SER_TYPES:
472473
# set service_data obj e.g self.linz_wms=service_data obj
473474
data_feed='{0}_{1}'.format(domain, service) # eg linz_wms
474475
setattr(self, data_feed, ServiceData(domain,
475476
service,
476477
self.service_versions,
477-
self.api_key_instance))
478+
self.api_key_instance,
479+
update_cache))
478480
service_data_instance=getattr(self, data_feed)
479481
self.data_feeds[data_feed]=service_data_instance #keep record of ser data insts
480482
service_data_instance.processServiceData()
@@ -485,6 +487,9 @@ def loadAllServices(self):
485487
all_data.extend(service_data_instance.info)
486488
self.table_model.setData(all_data)
487489
self.services_loaded=True
490+
491+
if update_cache:
492+
self.purgeCache()
488493
return None
489494

490495
def showSelectedOption(self, item):
@@ -539,7 +544,7 @@ def getPreview(self, res, res_timeout):
539544
url=('http://koordinates-tiles-d.global.ssl.fastly.net'
540545
'/services/tiles/v4/thumbnail/layer={0},style=auto/{1}.png'.format(self.id, res))
541546
try:
542-
img_data=urllib.request.urlopen(url, timeout=res_timeout).read()
547+
img_data=urlopen(url, timeout=res_timeout).read()
543548
except:
544549
return False
545550
self.qimage.loadFromData(img_data)
@@ -611,7 +616,7 @@ def filterTable(self):
611616
filter_text=self.dlg.uTextFilter.text()
612617
self.proxy_model.setFilterCaseSensitivity(Qt.CaseInsensitive)
613618
self.proxy_model.setFilterKeyColumn(2)
614-
self.proxy_model.setFilterRegExp(filter_text)
619+
self.proxy_model.setFilterFixedString(filter_text)
615620

616621
def setTableModelView(self):
617622
"""

‎linz-data-importer/service_data.py‎

Lines changed: 67 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,20 @@
1616
"""
1717

1818
import re
19+
import time
20+
import glob
21+
22+
import os
23+
from xml.etree.ElementTree import ParseError
24+
from urllib2 import urlopen, URLError
25+
1926
from owslib.wms import WebMapService
2027
from owslib.wfs import WebFeatureService
2128

2229
from owslib.wmts import WebMapTileService
2330
from owslib.util import ServiceException
2431
from qgis.core import QgsMessageLog, QgsApplication
2532

26-
import os.path
27-
from xml.etree.ElementTree import ParseError
28-
from urllib2 import urlopen, URLError
29-
3033
from PyQt4.QtCore import QSettings
3134
#from gc import isenabled
3235

@@ -100,7 +103,9 @@ def __init__(self, domain=None, service=None, file=None):
100103
self.file=file
101104
if self.service:
102105
self.file = os.path.join(self.pl_settings_dir ,
103-
'{0}_{1}.xml'.format(domain, service.lower()))
106+
'{0}_{1}_{2}.xml'.format(domain,
107+
service.lower(),
108+
time.strftime("%Y%m%d%H%M%S")))
104109

105110
def ensureSettingsDir(self):
106111
"""
@@ -133,6 +138,7 @@ def delDomainsXML(self, domain):
133138
if not domain:
134139
domain = self.domain
135140

141+
# Should use glob for consistency and simplicity
136142
dir = self.pl_settings_dir
137143
for f in os.listdir(self.pl_settings_dir):
138144
if re.search(domain, f):
@@ -149,24 +155,66 @@ def delAllLocalServiceXML(self, services=['wms','wfs','wmts']):
149155

150156
search_str = '|'.join(['_{}.xml'.format(x) for x in services])
151157

158+
# Should use glob for consistency and simplicity
152159
dir = self.pl_settings_dir
153160
for f in os.listdir(self.pl_settings_dir):
154161
if re.search(search_str, f):
155162
file = os.path.join(dir, f)
156163
self.delLocalSeviceXML(file)
157164

165+
def purgeCache(self):
166+
"""
167+
Delete all cached documents but the
168+
most current
169+
"""
170+
file_metadata={}
171+
os.chdir(self.pl_settings_dir)
172+
cache_files = glob.glob('*_*_[0-9]*.xml')
173+
174+
# Get cache metadata
175+
for f in cache_files:
176+
file_data=re.search(r'(?P<domain>.*)(_)(?P<service>wms|wmts|wfs)(_)(?P<time>[0-9]+)\.xml', f)
177+
domain=file_data.group('domain')
178+
service=file_data.group('service')
179+
timestamp=file_data.group('time')
180+
if domain not in file_metadata:
181+
file_metadata[domain]={}
182+
if service not in file_metadata[domain]:
183+
file_metadata[domain][service]=[]
184+
file_metadata[domain][service].append(timestamp)
185+
186+
# get list of most current
187+
curr_files = []
188+
for dom, v in file_metadata.iteritems():
189+
for ser , file_times in v.iteritems():
190+
curr_files.append('{0}_{1}_{2}.xml'.format(dom,
191+
ser,
192+
sorted(file_times)[-1]))
193+
194+
# del old files
195+
for file in cache_files:
196+
if file not in curr_files:
197+
self.delLocalSeviceXML(file)
198+
158199
def serviceXmlIsLocal(self, file=None):
159200
"""
160201
Test if the cached capabilties xml doc exists
202+
and if so set the services file to match this
203+
161204
:param file: file name
162205
:type file: str
163206
@return: boolean. True file exists
164207
@rtype: boolean
165208
"""
166209

167210
if not file:
168-
file = self.file
169-
return os.path.isfile(file)
211+
os.chdir(self.pl_settings_dir)
212+
files = glob.glob('{0}_{1}_*.xml'.format(self.domain,
213+
self.service.lower()))
214+
if files:
215+
self.file = sorted(files)[-1]
216+
return True
217+
return False
170218

171219
def readLocalServiceXml(self, file=None):
172220
"""
@@ -186,7 +234,7 @@ class ServiceData(Localstore):
186234
Get, Store and Process WxS Data
187235
"""
188236

189-
def __init__(self, domain, service, service_version, api_key_instance):
237+
def __init__(self, domain, service, service_version, api_key_instance, upd_cache=False):
190238
"""
191239
Initialise Service Data instance
192240
@@ -202,6 +250,7 @@ def __init__(self, domain, service, service_version, api_key_instance):
202250

203251
self.version = service_version[service]
204252
self.api_key_int = api_key_instance # using one instance as the user can change keys on us
253+
self.upd_cache=upd_cache
205254
Localstore.__init__(self, domain, service)
206255
# Data
207256
self.obj = None #owslib data obj
@@ -231,7 +280,13 @@ def getServiceData(self):
231280
Either via localstore or internet
232281
"""
233282

234-
# Get service xml
283+
# If cache get new data and overwrite local store
284+
if self.upd_cache:
285+
self.getServiceXml()
286+
return
287+
288+
# Plugin opened for first time
289+
# Read data from local store if exists
235290
if self.serviceXmlIsLocal():
236291
self.readLocalServiceXml()
237292
else:
@@ -240,10 +295,10 @@ def getServiceData(self):
240295
def getServiceDataTryAgain(self):
241296
"""
242297
If the reading of the capability doc fails - Try again.
243-
This is for use cases where for some reason the user
244-
has corrupted the capability doc in the localstore
298+
This is for use cases where for some reason the capability
299+
doc in the localstore has become corrupted
245300
"""
246-
301+
247302
#Clear error, Delete local file and get it a fresh
248303
self.err=None
249304
self.delLocalSeviceXML()

0 commit comments

Comments
 (0)