Python Cluster.setSplitpoint方法代码示例

# 需要导入模块: from cluster import Cluster [as 别名]
# 或者: from cluster.Cluster import setSplitpoint [as 别名]
def perform_recursive_clustering(cluster_collection, startAt):
    """
    Performs a recursive clustering on a list of clusters given via cluster_collection.
    The recursion is performed according to the Discoverer paper by Cui et al.
    At first new number of distinct values for each token are calculated in each cluster and
    if this number is lower than a configurable number, the token is considered a FD.
    Then the number of subclusters that would be generated is calculated. If these subclusters
    contain at least one cluster containing more than a configurable amount of messages, the clustering
    is performed and the token is considered a FD. Then the recursion is performed on each of the new clusters
    with the next token.
    
    
    """
    
    # Scan for FD token, Phase 1
    clusters = cluster_collection.get_all_cluster()[:] # <-- "[:]" Very very important... otherwise our iterated list will change because of deletions...
    
    # Save startAt information over cluster iteration
    __startAt = startAt
     
    for cluster in clusters:
        if Globals.getConfig().debug:
            print "Starting processing for next cluster ({0} messages)".format(len(cluster.get_messages()))
        
        startAt = __startAt
        #tokenValue = token.get_token()
        # Check distinct number of values of token
        foundFD = False
        maxTokenIdx = len(cluster.get_messages()[0].get_tokenlist())
        while not foundFD and startAt<maxTokenIdx:
            l = []
            #print "Analyzing token %s" % startAt
            # Check whether this might be a length token
            if "lengthfield" in set(cluster.get_semantics_for_token(startAt)):
                # Current token is a length token. Do not treat as FD
                startAt += 1
                continue
            if not Globals.getConfig().allowAdjacentFDs:
                if startAt>0:
                    if "FD"in set(cluster.get_semantics_for_token(startAt-1)): # We have an adjacent FD
                        print "Two adjacent FDs forbidden by configuration, skipping to next token"
                        continue
            
            for message in cluster.get_messages():
                l.append(message.get_tokenAt(startAt).get_token())
            numOfDistinctValuesForToken = len(set(l))
            
            if Globals.getConfig().minDistinctFDValues < numOfDistinctValuesForToken <= Globals.getConfig().maxDistinctFDValues:
                # FD candidate found
                # Check number of potential clusters
                sumUp = Counter(l)
                wouldCluster = False
                for key in sumUp.keys():
                    if sumUp.get(key)>Globals.getConfig().minimumClusterSize: # Minimum cluster size of at least one cluster
                        wouldCluster = True
                        break
                if wouldCluster:
                    # Check if adjacent text/text FDs are allowed in text protocols
                    if Globals.getProtocolClassification()==Globals.protocolText:
                        if not Globals.getConfig().allowAdjacentTextFDs:
                            if startAt>0:
                                # Check whether the previous one is a text FD (type text and no semantic numeric)
                                if "FD" in set(cluster.get_semantics_for_token(startAt-1)):
                                    if cluster.get_format(startAt-1)==Message.typeText and (
                                        cluster.get_format(startAt)==Message.typeText and ("numeric" not in cluster.get_semantics_for_token(startAt-1))):
                                        print "Two adjacent text FDs forbidden by configuration, skipping to next token"
                                        continue
                    # Create new cluster
                    if Globals.getConfig().debug:
                        print "Subcluster prerequisites fulfilled. Adding FD semantic, splitting cluster and entering recursion"
                    # Senseless here: message.get_tokenAt(startAt).add_semantic("FD")
                    cluster.add_semantic_for_token(startAt,"FD")
                    newCollection = ClusterCollection()
                    for key in sumUp.keys():
                            messagesWithValue = cluster.get_messages_with_value_at(startAt,key)
                            newCluster = Cluster(messagesWithValue[0].get_tokenrepresentation(), "recursion")
                            newCluster.setSplitpoint("{0}".format(startAt))
                            newCluster.add_messages(messagesWithValue)                            
                            newCluster.add_semantic_for_token(startAt, "FD")
                            newCollection.add_cluster(newCluster)
                    if Globals.getConfig().debug:
                        print "{0} sub clusters generated".format(len(sumUp.keys()))
                    
                    # Perform format inference on new cluster collection
                    formatinference.perform_format_inference_for_cluster_collection(newCollection)
                    semanticinference.perform_semantic_inference(newCollection)
                    
                    # Merge clusters with same format
                    while newCollection.mergeClustersWithSameFormat():
                        pass
                    
                    # Perform needle wunsch
                    # Edit 20120120 - not here
                    #===========================================================
                    # cluster1 = newCollection.get_random_cluster()
                    # cluster2 = newCollection.get_random_cluster()
                    # format1 = cluster1.get_formats()
                    # format2 = cluster2.get_formats()
                    # needlewunsch.needlewunsch(format1, format2)
                    # 
#.........这里部分代码省略.........

# 需要导入模块: from cluster import Cluster [as 别名]
# 或者: from cluster.Cluster import setSplitpoint [as 别名]

#.........这里部分代码省略.........
            #for idx_inner in range(1,len(copiedCollection)-1):    
                
                cluster2 = copiedCollection[idx_inner]
                format1 = cluster1.get_formats()
                format2 = cluster2.get_formats()
                if not len(format1)==len(format2):
                    idx_inner += 1
                    continue # The two clusters have different length [should not happen within subclusters]
                # Perform token check
                shouldMerge = True
                for format_token_idx in range(0,len(format1)-1):
                    token1 = cluster1.get_format(format_token_idx)
                    token2 = cluster2.get_format(format_token_idx)
                    representation = token1[0]
                    fmt_infer = token1[1]
                    semantics = token1[2]
                    if not representation == token2[0]: # Token mismatch --> will not merge
                        shouldMerge = False
                        break
                    
                    checkValues = False
                    if semantics == token2[2]:
                        if len(semantics)==0: # They match because there are no semantics... :-(
                            checkValues = True 
                    else: # Semantics mismatch --> will not merge
                        shouldMerge = False
                        break
                    
                    
                    if checkValues:
                        if fmt_infer.getType() == token2[1].getType():
                            # Check constant/variable cover
                            if fmt_infer.getType()=='const': 
                                # Check instance of const value
                                # FIX: Each cluster must have at least 1 message!
                                if not cluster1.get_messages()[0].get_tokenAt(format_token_idx).get_token() == cluster2.get_messages()[0].get_tokenAt(format_token_idx).get_token():
                                    # Const value mismatch --> will not merge
                                    shouldMerge = False
                                    break
                            else:
                                # Check variable/variable instances
                                # Check for overlap in values. If there is no overlap -> Mismatch
                                allvalues1 = cluster1.get_values_for_token(format_token_idx)
                                allvalues2 = cluster2.get_values_for_token(format_token_idx)
                                if len(set(allvalues1).intersection(set(allvalues2)))==0:
                                    # No overlap -> Mismatch
                                    shouldMerge = False
                                    break
                            
                        else:
                            # Variable/Constant format inference
                            # Check whether variable token takes value of constant one at least once
                            found = True
                            if fmt_infer.getType() == 'const':
                                # Search for cluster1's value in cluster2
                                cluster1val = cluster1.get_messages()[0].get_tokenAt(format_token_idx).get_token()
                                hits = cluster2.get_messages_with_value_at(format_token_idx,cluster1val)
                                found = len(hits)>0
                            else:
                                # Search for cluster2's value in cluster1
                                cluster2val = cluster2.get_messages()[0].get_tokenAt(format_token_idx).get_token()
                                hits = cluster1.get_messages_with_value_at(format_token_idx,cluster2val)
                                found = len(hits)>0
                            if not found:
                                # No instance of variable in const mismatch --> will not merge
                                shouldMerge = False
                                break
            
            
                               
                # End of token iteration
                if shouldMerge:    
                    mergeCandidates.append(cluster2)
                idx_inner += 1     
            # End of for each clusterloop
            
            newCluster = Cluster(cluster1.get_representation(), "mergeDestination")
            newCluster.set_semantics(cluster1.get_semantics())             
            newCluster.add_messages(cluster1.get_messages())
            splitpoint = ""
            for cluster in mergeCandidates:                    
                newCluster.add_messages(cluster.get_messages())
                copiedCollection.remove(cluster)
                splitpoint = "{0}, {1}".format(splitpoint, cluster.getSplitpoint())
            newCluster.setSplitpoint(splitpoint)
            discoverer.formatinference.perform_format_inference_for_cluster(newCluster)    
            # TODO: Build up new semantic information in newCluster
            copiedCollection.remove(cluster1)               
            tempCollection.add_cluster(newCluster)            
                
        # Clear own collection
        self.__cluster = []
        # Copy all clusters from tempCollection to our self
        self.add_clusters(tempCollection.get_all_cluster())
        if ori_len == len(self.__cluster):
            logging.info("No mergable clusters within collection identified")
            return False
        else:
            logging.info("Cluster collection shrunk from {0} to {1} by merging".format(ori_len, len(self.__cluster)))
            return True