Python LDSUtilities.LDSUtilities类代码示例

 def buildIndex(self):
     '''Builds an index creation string for a new full replicate in PG format'''
     tableonly = self.dst_info.ascii_name.split('.')[-1]
     ALLOW_TABLE_INDEX_CREATION=True
     #SpatiaLite doesnt have a unique constraint but since we're using a pk might a well declare it as such
     if ALLOW_TABLE_INDEX_CREATION and LDSUtilities.assessNone(self.dst_info.pkey):
         #spatialite won't do post create constraint additions (could to a re-create?)
         cmd = 'CREATE INDEX {0}_{1}_PK ON {0}({1})'.format(tableonly,self.dst_info.pkey)
         try:
             self.executeSQL(cmd)
             ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.pkey,cmd))
         except RuntimeError as rte:
             if re.search('already exists', str(rte)): 
                 ldslog.warn(rte)
             else:
                 raise        
     
     #Unless we select SPATIAL_INDEX=no as a Layer option this should never be needed
     #because gcol is also used to determine whether a layer is spatial still do this check   
     if LDSUtilities.assessNone(self.dst_info.geocolumn):
         #untested and unlikely to work
         cmd = "CREATE INDEX {0}_{1}_SK ON {0}({1})".format(self.dst_info.ascii_name,self.dst_info.geocolumn)
         try:
             self.executeSQL(cmd)
             ldslog.info("Index = {}({}). Execute = {}.".format(tableonly,self.dst_info.geocolumn,cmd))
         except RuntimeError as rte:
             if re.search('already exists', str(rte)): 
                 ldslog.warn(rte)
             else:
                 raise

 def buildIndex(self):
     '''Builds an index creation string for a new full replicate in PG format'''
     tableonly = self.dst_info.ascii_name.split('.')[-1]
     
     if LDSUtilities.assessNone(self.dst_info.pkey):
         cmd = 'ALTER TABLE {0} ADD CONSTRAINT {1}_{2}_PK UNIQUE({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.pkey)
         try:
             self.executeSQL(cmd)
             ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.pkey,cmd))
         except RuntimeError as rte:
             if re.search('already exists', str(rte)): 
                 ldslog.warn(rte)
             else:
                 raise
                     
     #If a spatial index has already been created don't try to create another one
     if self.SPATIAL_INDEX == 'OFF' and LDSUtilities.assessNone(self.dst_info.geocolumn):
         cmd = 'CREATE INDEX {1}_{2}_GK ON {0} USING GIST({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.geocolumn)
         try:
             self.executeSQL(cmd)
             ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.geocolumn,cmd))
         except RuntimeError as rte:
             if re.search('already exists', str(rte)): 
                 ldslog.warn(rte)
             else:
                 raise

 def buildIndex(self):
     '''Builds an index creation string for a new full replicate in PG format'''
     tableonly = self.dst_info.ascii_name.split('.')[-1]
     ALLOW_CONSTRAINT_CREATION=False
     #SpatiaLite doesnt have a unique constraint but since we're using a pk might a well declare it as such
     if ALLOW_CONSTRAINT_CREATION and LDSUtilities.assessNone(self.dst_info.pkey):
         #spatialite won't do post create constraint additions (could to a re-create?)
         cmd = 'ALTER TABLE {0} ADD PRIMARY KEY {1}_{2}_PK ({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.pkey)
         try:
             self.executeSQL(cmd)
             ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.pkey,cmd))
         except RuntimeError as rte:
             if re.search('already exists', str(rte)): 
                 ldslog.warn(rte)
             else:
                 raise        
     
     #Unless we select SPATIAL_INDEX=no as a Layer option this should never be needed
     #because gcol is also used to determine whether a layer is spatial still do this check   
     if LDSUtilities.assessNone(self.dst_info.geocolumn) and 'SPATIAL_INDEX=NO' in [opt.replace(' ','').upper() for opt in self.sl_local_opts]:
         cmd = "SELECT CreateSpatialIndex('{}','{}')".format(self.dst_info.ascii_name,self.DEFAULT_GCOL)
         try:
             self.executeSQL(cmd)
             ldslog.info("Index = {}({}). Execute = {}. NB Cannot override Geo-Column Name.".format(tableonly,self.DEFAULT_GCOL,cmd))
         except RuntimeError as rte:
             if re.search('already exists', str(rte)): 
                 ldslog.warn(rte)
             else:
                 raise

    def testURL(self,url):
        '''Connect to a URL using the configured proxy (using urlopen method)'''
        return LDSUtilities.timedProcessRunner(LDSUtilities.readLDS, (url,self.pxy), None)
        #return LDSUtilities.readDocument(url, self.pxy)
        

        

 def fetchLayerInfo(cls,url,ver=None,proxy=None):
     '''Non-GDAL static method for fetching LDS layer ID's using etree parser.'''
     res = []
     content = None
     wfs_ns = cls.NS['wfs20'] if re.match('^2',ver) else cls.NS['wfs11']
     ftxp = "//{0}FeatureType".format(wfs_ns)
     nmxp = "./{0}Name".format(wfs_ns)
     ttxp = "./{0}Title".format(wfs_ns)
     kyxp = "./{0}Keywords/{0}Keyword".format(cls.NS['ows'])
     
     try:            
         if not LDSUtilities.assessNone(proxy): install_opener(build_opener(ProxyHandler(proxy)))
         #content = urlopen(url)#bug in lxml doesnt close url/files using parse method
         with closing(urlopen(url)) as content:
             tree = etree.parse(content)
             for ft in tree.findall(ftxp):
                 name = ft.find(nmxp).text#.encode('utf8')
                 title = ft.find(ttxp).text#.encode('utf8')
                 #keys = [x.text.encode('utf8') for x in ft.findall(kyxp)]
                 keys = [x.text for x in ft.findall(kyxp)]
                 
                 res += ((name,title,keys),)
             
     except XMLSyntaxError as xe:
         ldslog.error('Error parsing URL;'+str(url)+' ERR;'+str(xe))
         
     return res

 def getLayerNames(self,refresh=False):
     '''Returns configured layers for respective layer properties file'''
     #gdal.SetConfigOption('CPL_DEBUG','ON')
     #gdal.SetConfigOption('CPL_LOG_ERRORS','ON')
     if refresh or not self.namelist:
         self.namelist = []
         layer = self.lcfname.ds.GetLayer(self.lcfname.LDS_CONFIG_TABLE)###fname -> lcfname
         if layer:
             layer.SetIgnoredFields(('OGR_GEOMETRY',))
             layer.ResetReading()
             #HACK Win7
             layer.GetFeatureCount()
             feat = layer.GetNextFeature() 
             while feat:
                 try:
                     lcid = feat.GetField('id')
                     lcname = feat.GetField('name')
                     lccats = [LU.recode(f) if f else None for f in feat.GetField('category').split(',')]
                     self.namelist += ((LU.recode(lcid) if lcid else None, LU.recode(lcname) if lcname else None,lccats),) 
                     #self.namelist += ((feat.GetField('id').encode('utf8'),feat.GetField('name').encode('utf8'),[f.encode('utf8').strip() for f in feat.GetField('category').split(',')]),)
                     feat = layer.GetNextFeature()
                 except UnicodeEncodeError as uee:
                     raise
                 except UnicodeDecodeError as ude:
                     raise
         else:
             ldslog.error('REMINDER! TRIGGER CONF BUILD')
         #print '>>>>> NAMELIST',self.namelist
     return self.namelist

 def buildIndex(self):
     '''Builds an index creation string for a new full replicate'''
     tableonly = self.dst_info.ascii_name.split('.')[-1]
     
     if LU.assessNone(self.dst_info.pkey):
         cmd = 'ALTER TABLE {0} ADD CONSTRAINT {1}_{2}_PK UNIQUE({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.pkey)
         try:
             self.executeSQL(cmd)
             ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.pkey,cmd))
         except RuntimeError as rte:
             if re.search('already exists', str(rte)): 
                 ldslog.warn(rte)
             else:
                 raise        
             
     if LU.assessNone(self.dst_info.geocolumn):
         cmd = 'CREATE SPATIAL INDEX {1}_{2}_GK ON {0}({2})'.format(self.dst_info.ascii_name,tableonly,self.dst_info.geocolumn)
         cmd += ' WITH (BOUNDING_BOX = (XMIN = {0},YMIN = {1},XMAX = {2},YMAX = {3}))'.format(self.BBOX['XMIN'],self.BBOX['YMIN'],self.BBOX['XMAX'],self.BBOX['YMAX'])
         #cmd = 'CREATE SPATIAL INDEX ON {}'.format(tableonly)
         try:
             self.executeSQL(cmd)
             ldslog.info("Index = {}({}). Execute = {}".format(tableonly,self.dst_info.geocolumn,cmd))
         except RuntimeError as rte:
             if re.search('already exists', str(rte)): 
                 ldslog.warn(rte)
             else:
                 raise

 def buildLayerGroupList(self,groups=None,layers=None):
     '''Sets the array storing values displayed in the Layer/Group combo; format is ((lyr/grp, name, displaystring),)'''
     from lds.TransferProcessor import LORG
     #self.lgcombo.addItems(('',TransferProcessor.LG_PREFIX['g']))
     self.lglist = []
     #              lorg,value,display
     for g in sorted(groups if groups else self.assigned):
         self.lglist += ((LORG.GROUP,LU.recode(g),u'{} (group)'.format(LU.recode(g))),)
     for l in sorted(layers if layers else self.vlayers):
         #if re.search('Electoral',l[0]): 
         if not isinstance(l,(tuple,list)): raise LayerConfigInitialisationException('Layer init value error. {} is not a list/tuple'.format(l))
         self.lglist += ((LORG.LAYER,l[0],u'{} ({})'.format(l[1],l[0])),)

    def runReplicationScript(self,clean=False):
        '''Run the layer/group repliction script'''
        destination,lgval,uconf,epsg,fe,te,fd,td = self.readParameters()
        uconf_path = LU.standardiseUserConfigName(uconf)
        destination_path = LU.standardiseLayerConfigName(destination)
        destination_driver = LU.standardiseDriverNames(destination)

        if not os.path.exists(uconf_path):
            self.userConfMessage(uconf_path)
            return
        elif not MainFileReader(uconf_path).hasSection(destination_driver):
            self.userConfMessage(uconf_path,destination_driver)
            return
        #-----------------------------------------------------
        #'destname','layer','uconf','group','epsg','fd','td','int'
     
        self.parent.gpr.write((destination_driver,lgval,uconf,epsg,fd,td))        
        ldslog.info(u'dest={0}, lg={1}, conf={2}, epsg={3}'.format(destination_driver,lgval,uconf,epsg))
        ldslog.info('fd={0}, td={1}, fe={2}, te={3}'.format(fd,td,fe,te))
        lgindex = self.parent.confconn.getLayerGroupIndex(lgval,col=1)
        #lorg = self.parent.confconn.lglist[lgindex][0]
        #----------don't need lorg in TP anymore but it is useful for sorting/counting groups
        #self.parent.confconn.tp.setLayerOrGroup(lorg)
        self.parent.confconn.tp.setLayerGroupValue(lgval)
        if self.fromdateenable.isChecked(): self.parent.confconn.tp.setFromDate(fd)
        if self.todateenable.isChecked(): self.parent.confconn.tp.setToDate(td)
        self.parent.confconn.tp.setUserConf(uconf)
        if self.epsgenable: self.parent.confconn.tp.setEPSG(epsg)
        
        #because clean state persists in TP
        if clean:
            self.parent.confconn.tp.setCleanConfig()
        else:
            self.parent.confconn.tp.clearCleanConfig()
        #(re)initialise the data source since uconf may have changed
        #>>#self.parent.confconn.tp.src = self.parent.confconn.initSourceWrapper()
        #--------------------------
        ###ep = self.parent.confconn.reg.openEndPoint(self.parent.confconn.destname,self.parent.confconn.uconf)
        
        ###self.parent.confconn.reg.closeEndPoint(self.parent.confconn.destname)
        ###ep = None
        #Open ProcessRunner and run with TP(proc)/self(gui) instances
        #HACK temp add of dest_drv to PR call
        try:
            #TODO. Test for valid LC first
            self.tpr = ProcessRunner(self,destination_driver)
        except Exception as e:
            ldslog.error('Cannot create ProcessRunner {}. NB Possible missing Layer Config {}'.format(str(e),destination_path))
            self.layerConfMessage(destination_path)
            return
        #If PR has been successfully created we must vave a valid dst    
        ldslog.debug('TRPstart')
        self.tpr.start()

 def readLayerProperty(self,layer,key):
     try:
         if isinstance(layer,tuple) or isinstance(layer,list):
             value = () 
             for l in layer:
                 value += (LU.assessNone(self.cp.get(l, key)),)
         else:
             value = LU.assessNone(self.cp.get(layer, key))
     except:
         '''return a default value otherwise none which would also be a default for some keys'''
         #the logic here may be a bit suss, if the property is blank return none but if there is an error assume a default is needed?
         return {'pkey':'ID','name':layer,'geocolumn':'SHAPE'}.get(key)
     return value

 def getLayerGroupIndex(self,dispval,col=2):
     '''Finds a matching group/layer entry from its displayed name'''
     #0=lorg,1=value,2=display 
     compval = LU.recode(dispval)
     if not LU.assessNone(compval):
         ldslog.warn('No attempt made to find index for empty group/layer request, "{}"'.format(compval))
         return None# or 0?
         
     try:
         #print 'lgl',[type(i[1]) for i in self.lglist],'\ncv',type(compval)
         index = [i[col] for i in self.lglist].index(compval)
     except ValueError as ve:
         ldslog.warn(u'Cannot find an index in column {} for the requested group/layer, "{}", from {} layers. Returning None index'.format(col,compval,len(self.lglist)))
         index = None
     return index

    def deleteKeysFromLayerConfig(self,layerlist,ckey):
        '''Remove custom keys from selected layers'''
        replacementlist = ()
        dep = self.parent.confconn.reg.openEndPoint(self.parent.confconn.destname,self.parent.confconn.uconf)
        #print 'opening',dep#DEBUG
        self.parent.confconn.reg.setupLayerConfig(self.parent.confconn.tp,None,dep,initlc=False)
        categorylist = [f.encode('utf8').strip() for f in dep.getLayerConf().readLayerProperty(layerlist, 'category') if f]
        for cat in categorylist:
            #replacementlist += (re.sub(',+',',',''.join(cat.split(ckey)).strip(',')),)
            try:  
                cat = cat.split(',')
                cat.remove(LU.recode(ckey,uflag='encode'))
            except ValueError:
                pass
            replacementlist += (','.join(cat),)
        dep.getLayerConf().writeLayerProperty(layerlist, 'category', replacementlist)
 
        #-----------------------------------
        self.parent.confconn.setupCompleteLayerList(dep)
        self.parent.confconn.setupAssignedLayerList()
        self.parent.confconn.buildLayerGroupList()   
        #self.refreshLayers(dep,customkey)
        self.parent.confconn.reg.closeEndPoint(self.parent.confconn.destname)
        #print 'closing', dep,self.parent.confconn.reg#DEBUG
        dep = None
        return self.selection_model.rowCount()

 def doRejectClickAction(self):
     '''Takes available selected and moves to selection'''
     #ktext = LU.recode(self.keywordcombo.lineEdit().text().toUtf8().data())
     ktext = LU.recode(LQ.readWidgetText(self.keywordcombo.lineEdit().text()))
     if not self.checkKeyword(ktext): return
     #------------------------------
     select = self.selection.selectionModel()
     if select.hasSelection():
         tlist = self.transferSelectedRows(select.selectedRows(),self.selection_sfpm,self.available_sfpm)
         #------------------------------
         kindex = self.keywordcombo.findText(ktext)
         remainder = self.parent.deleteKeysFromLayerConfig([ll[1][0] for ll in tlist],ktext)
         if remainder > 0 and kindex == -1:
             #items+newkey -> add
             self.parent.writeKeysToLayerConfig(ktext)
             self.keywordcombo.addItem(ktext)
         elif remainder == 0 and kindex > -1:
             #empty+oldkey -> del
             self.keywordcombo.removeItem(kindex)
             self.keywordcombo.clearEditText()
     else:
         ldslog.warn('R2L < Transfer action without selection')
     #TRACE#
     #pdb.set_trace()
     self.selection.clearSelection()

 def writesecline(self,section,field,value):
     try:            
         self.cp.set(section,field,value if LU.assessNone(value) else '')
         with codecs.open(self.fn, 'w','utf-8') as configfile:
             self.cp.write(configfile)
         ldslog.debug(str(section)+':'+str(field)+'='+str(value))                                                                                        
     except Exception as e:
         ldslog.warn('Problem writing GUI prefs. {} - sfv={}'.format(e,(section,field,value)))

 def _deregister(self,name):
     fn = LU.standardiseDriverNames(name)
     #sync/rel the DS
     #self.register[fn]['ep'].closeDS()#DS should already have closed
     self.register[fn]['ep'] = None
     self.register[fn]['type'] = None
     self.register[fn]['uri'] = None
     del self.register[fn]