Python Session.flush方法代碼示例

# 需要導入模塊: from hpf.hddb.db import Session [as 別名]
# 或者: from hpf.hddb.db.Session import flush [as 別名]
def translate_foldables(dir, update_db=True):
    """Translates foldable fasta files in the manner described above. If update_db is true,
    adds new sequences translated to hpf.sequence table and updates hpf.filesystemOutfile 
    (foldable records) to link to the new sequence key.
    """
    from hashlib import sha1
    from hpf.hddb.reexport_foldables import write_fasta
    from hpf.hddb.db import push_to_db, Session, Sequence, FilesystemOutfile
   
    print "Translating foldable fastas found in dir {0}".format(dir)
    
    files = os.listdir(dir)
    for file in files:
        try: 
            code = parse_code_from_file(file)
        except IOError as e:
            print "{0}. Ignoring file..".format(e)
        sequence_key, sequence = parse_foldable_file(file)

        # If the sequence contains a non-standard AA code, translate nonstandard to normal codes
        if re.search(nonstandard_pattern, sequence):
            print "{0} contains nonstandard AA codes. Translating".format(file)
            print "\tOriginal  : {0}".format(sequence)
            
            translated_seq_id = "None"
            for nsaa in non_standard.keys():
                sequence = sequence.replace(nsaa, non_standard[nsaa])
            
            print "\tTranslated: {0}".format(sequence)
            
            if update_db:
                # Add new sequence to DB (push_ will return None if seq_dbo is already in DB)
                print "Adding translated sequence to the DB"
                session = Session()
                seq_dbo = Sequence(sequence=sequence, sha1=sha1(sequence).hexdigest())
                seq_dbo = push_to_db(session, seq_dbo, exception_str="Pushing sequence for code {0} failed".format(code), raise_on_duplicate=False)
                if not seq_dbo:
                    seq_dbo = session.query(Sequence).filter_by(sha1=sha1(sequence).hexdigest()).first()
                
                # Get foldable record and change seq key to new translated seq's id
                print "Updating foldable record from old ({0}) to new ({1}) sequence key".format(sequence_key, seq_dbo.id)
                foldable_dbo = session.query(FilesystemOutfile).filter_by(prediction_code=code).first()
                if not foldable_dbo:
                    raise Exception("Foldable record not found in DB for code {0}".format(code))
                foldable_dbo.sequence_key = seq_dbo.id
                session.flush()
                
                translated_seq_id = seq_dbo.id
            
            print "Writing translated foldable to file {0}".format(file)
            with open(file, 'w') as handle:
                write_fasta(handle, translated_seq_id, len(sequence), sequence)
    print "Translating foldables complete"

# 需要導入模塊: from hpf.hddb.db import Session [as 別名]
# 或者: from hpf.hddb.db.Session import flush [as 別名]
class OIDImporter(object):
    """
    Import a set of OID files into the database
    """

    def __init__(
        self,
        familyName,
        alignFile,
        alignColcullLog,
        alignSeqcullLog,
        treeFile,
        treeDiagCharsFile,
        codemlFile=None,
        alignFormat="fasta",
        oid_key=None,
    ):
        self.familyName = familyName
        self.treeFile = treeFile
        self.treeDiagCharsFile = treeDiagCharsFile
        self.alignFile = alignFile
        self.alignColcullLog = alignColcullLog
        self.alignSeqcullLog = alignSeqcullLog
        self.codemlFile = codemlFile
        self.alignFormat = alignFormat
        self.oid_key = oid_key

    def merge(self):
        from hpf.hddb.db import Session, Family

        self.session = Session()

        self.family = self.session.query(Family).filter(Family.name == self.familyName).first()
        if not self.family:
            runtime().debug("Creating family", self.familyName)
            self._family()
            self._alignment()
            self._tree()
        else:
            self.alignment = self.family.alignment
            self.tree = self.alignment.tree
            runtime().debug("Found family", self.family.id)

        if not self.family.alignments[0].tree.codeml:
            runtime().debug("Importing codeml")
            self._codeml()
        else:
            runtime().debug("Already found codeml", self.family.alignments[0].tree.codeml.id)

        # Commit the session, close, and finish
        self.session.commit()
        self.session.close()

    def _index(self, name):
        n = name.split("#")[-1]
        if n.startswith("N"):
            n = n[1:]
        assert n.isdigit()
        return n

    def _tree(self):
        session = self.session

        # # Load the tree file and rename the taxa.
        # from Bio.Nexus.Nexus import Nexus
        # nex=Nexus(self.treeFile)
        # self.nexus = nex.trees[0]

        from Bio.Nexus.Trees import Tree as NewickTree

        tree_str = open(self.treeFile).read()
        self.nexus = NewickTree(tree_str)

        # Rename all the taxa.
        for id in self.nexus.get_terminals():
            node = self.nexus.node(id)
            node.data.taxon = self._index(node.data.taxon)

        # Create the DB object
        from hpf.hddb.db import Tree

        self.tree = Tree(
            alignment_key=self.alignment.id,
            text=self.nexus.to_string(plain=False, plain_newick=True),
            filename=self.treeFile,
        )
        session.add(self.tree)
        session.flush()

        # Now add in the node references
        self.nexus.name = self.tree.id
        assert self.tree.id != None
        runtime().debug("Added tree", self.tree)
        from hpf.hddb.db import TreeNodeFactory

        nodes = list(TreeNodeFactory().create(self.nexus))
        for node in nodes:
            node.ancestor_node = node.ancestor.id if node.ancestor else None
            # This should add the new object into the session
            self.tree.nodes.append(node)
#.........這裏部分代碼省略.........

# 需要導入模塊: from hpf.hddb.db import Session [as 別名]
# 或者: from hpf.hddb.db.Session import flush [as 別名]
        "--structure_type",
        action="store",
        dest="structure_type",
        required=True,
        help="Type of structure (astral, pdb, denovo, other)",
    )
    parser.add_argument(
        "--mammoth_table",
        action="store",
        dest="table_destination",
        default="default",
        help="The StructureMammoth table in hpf database to store mammoth results, default=StructureMammoth, 1186=yeast experiment",
    )

    args = parser.parse_args()

    record = (
        session.query(MammothRun)
        .filter(MammothRun.supergroup_key == args.supergroup_key)
        .filter(MammothRun.version == args.version)
        .filter(MammothRun.group_key1 == args.group_key1)
        .filter(MammothRun.group_key2 == args.group_key2)
        .filter(MammothRun.status == "running")
        .first()
    )
    store_scores(args.results_file, record, args.structure_type, args.version, args.table_destination)
    record.status = "complete"
    record.comment = "Mammoth run successful, imported results using store_scores.py"

    session.flush()

# 需要導入模塊: from hpf.hddb.db import Session [as 別名]
# 或者: from hpf.hddb.db.Session import flush [as 別名]
def main(letter_code, version):
    """
    Organizes the main functionality of getting a McmRun/Foldable to work on, extracting and checking
    the rosetta decoy file, and running MCM on the decoys. Everything after fetching the mcmRun DB
    record is in a try-catch block, to handle setting DB record status as well as possible.
    """
    from datetime import datetime
    from hpf.mcm.new_mcm import MCM
    from hpf.hddb.db import Session, McmRun, RosettaConvergence, RosettaCluster, Structure, Mammoth, McmData
    session = Session()

    # Get MCM/foldable record (whole code, eg ox971) to work on
    print "DRIVER: Fetching MCM Run record from DB..."
    mcm_records = fetch_mcmRun_records(session, letter_code, version)
    if not mcm_records:
        raise Exception("No McmRun records of code {0}, version {1} available".format(letter_code, version))
    for mcm_record in mcm_records:
        print "CLEANUP: Obtained MCM Run record {1} for foldable: {0}, status {2}".format(mcm_record.prediction_code, mcm_record.id, mcm_record.status)

        #kdrew: check for proper values in mcm tables
        rosetta_convergence_count = 0 #1 entry per mcm
        rosetta_cluster_count = 0 #normally 25 entries per mcm
        structure_count = 0 #normally 25 entries per mcm, should equal rosetta_cluster_count
        mammoth_count = 0 #normally 5 entries
        mcm_count = 0 #normally 5 entries, should equal mammoth_count

        rosetta_convergence_error = False
        rosetta_cluster_error = False
        structure_error = False
        mammoth_error = False
        mcm_error = False

        error_str = ""

        rosetta_convergence_count = session.query(RosettaConvergence).filter_by(outfile_key = mcm_record.foldable.id).count()
        rosetta_convergence = session.query(RosettaConvergence).filter_by(outfile_key = mcm_record.foldable.id).first()
        print rosetta_convergence, rosetta_convergence_count

        if rosetta_convergence:
            rosetta_cluster_count = session.query(RosettaCluster).filter_by(convergence_key=rosetta_convergence.id).count()
            rosetta_clusters = session.query(RosettaCluster).filter_by(convergence_key=rosetta_convergence.id).all()
            print rosetta_clusters[0], len(rosetta_clusters)
            if len(rosetta_clusters) == 0:
                rosetta_cluster_error = True
                error_str = error_str + " missing rosetta_cluster entry,"

            for rosetta_cluster in rosetta_clusters:
                structure = session.query(Structure).filter_by(id=rosetta_cluster.structure_key).first()
                #print structure
                if structure:
                    structure_count += 1
                else:
                    structure_error = True
                    error_str = error_str + " missing structure entry,"
        else:
            rosetta_convergence_error = True
            error_str = error_str + " missing rosetta_convergence entry,"
               
        print "structure_count: %s" % (structure_count,)

        mcm_entries = session.query(McmData).filter_by(outfile_key = mcm_record.foldable.id).all()
        if mcm_entries:
            mcm_count = len(mcm_entries)
            for mcm_entry in mcm_entries:
                print mcm_entry
                mammoth_entry = session.query(Mammoth).filter_by(p_structure_key = mcm_entry.structure_key).all()
                if mammoth_entry:
                    mammoth_count += 1
                else:
                    mammoth_error = True
                    error_str = error_str + " missing mammoth entry,"
        else:
            mcm_error = True
            error_str = error_str + " missing mcm entry,"

        print "mammoth_count: %s" % (mammoth_count,)

        #kdrew: if entries are in the db and no errors
        if not mammoth_error and not mcm_error and not rosetta_convergence_error and not rosetta_cluster_error and not structure_error:
            #kdrew: if counts are as expected
            if mammoth_count == 5 and mcm_count == 5 and rosetta_cluster_count == 25 and structure_count == 25:
                if mcm_record.status != 'complete': 
                    mcm_record.finished = datetime.now()
                    mcm_record.status  = 'complete'
                    mcm_record.comment = "MCM run successful (on mcmrun_cleanup_db)"
                    session.flush()
                #else: do not change completed runs
            #kdrew: missing db entries but no errors
            else:
                warning_str = ""
                if mammoth_count != 5:
                    warning_str = warning_str + " missing mammoth entries,"
                if mcm_count != 5:
                    warning_str = warning_str + " missing mcm entries,"
                if rosetta_cluster_count != 25:
                    warning_str = warning_str + " missing rosetta_cluster entries,"
                if structure_count != 25:
                    warning_str = warning_str + " missing structure entries,"

                mcm_record.finished = datetime.now()
#.........這裏部分代碼省略.........

# 需要導入模塊: from hpf.hddb.db import Session [as 別名]
# 或者: from hpf.hddb.db.Session import flush [as 別名]
def main(letter_code, version, host, work_base, results_dir, retry):
    """
    Organizes the main functionality of getting a McmRun/Foldable to work on, extracting and checking
    the rosetta decoy file, and running MCM on the decoys. Everything after fetching the mcmRun DB
    record is in a try-catch block, to handle setting DB record status as well as possible.
    """
    from datetime import datetime
    from hpf.mcm.new_mcm import MCM
    from hpf.hddb.db import Session, McmRun
    session = Session()

    # Get MCM/foldable record (whole code, eg ox971) to work on
    print "DRIVER: Fetching MCM Run record from DB..."
    mcm_record = fetch_mcmRun_record(session, letter_code, version, host, retry)
    if not mcm_record:
        raise Exception("No McmRun record of code {0}, version {1} available to run (batch may be complete!)".format(letter_code, version))
    print "DRIVER: Obtained MCM Run record {1} for foldable: {0}".format(mcm_record.prediction_code, mcm_record.id)
    
    # Set up environment: make working directory and change there
    try:
        work_dir = os.path.join(work_base, mcm_record.prediction_code)
        if not os.path.isdir(work_dir): os.mkdir(work_dir)
        os.chdir(work_dir)
        print "DRIVER: Working directory holding all temp files: {0}".format(work_dir)
    except Exception as e:
        set_error(session, mcm_record, e, error_str="Error creating environment")
        raise e

    # Find decoy file and unzip to working directory (Exception if can't be found in results dir)
    try:
        results_file = fetch_results_file(results_dir, mcm_record.prediction_code)
        if not results_file: 
            raise Exception("No results file for code {0} found in {1}".format(mcm_record.prediction_code, results_dir))
        decoy_file = "{0}.decoys".format(mcm_record.prediction_code)
        decompress_results(results_file, decoy_file)
        print "DRIVER: Found results file '{0}'. Unzipped to working directory".format(results_file)
    except Exception as e:
        set_error(session, mcm_record, e, error_str="Error processing results file")
        raise e
    
    # Initialize MCM object and set McmRun record start time
    print "DRIVER: Creating MCM object on denovo results file: '{0}'".format(decoy_file)
    try:
        mcm = MCM(code=mcm_record.prediction_code,
                  decoy_file=decoy_file,
                  work_dir=work_base,
                  rosetta_pathsfile=ROSETTA_PATHS,
                  mammoth_listfile=MAMMOTH_LIST,
                  mammoth_datafile=MAMMOTH_DATA,
                  ginzu_version=GINZU_VERSION,
                  ignore_ginzu_version=IGNORE_GINZU_VERSION,
                  nr_db=NR_DATABASE,
                  dbstore=DBSTORE,
                  cleanup=CLEANUP,
                  debug=DEBUG,
                 )
        mcm_record.started = datetime.now()
    except Exception as e:
        set_error(session, mcm_record, e, error_str="Error initializing MCM")
        raise e

    # Run MCM object
    print "DRIVER: Running MCM process (start time set)"
    try:
        mcm.run()
    except Exception as e:
        set_error(session, mcm_record, e, error_str="Error running MCM")
        traceback.print_exc(file=sys.stderr)
        raise e
    
    # On successful finish, set finished time and status (only complete if results in DB) and exit
    mcm_record.finished = datetime.now()
    if DBSTORE:
        mcm_record.status  = 'complete'
        mcm_record.comment = "MCM run successful"
    else: 
        mcm_record.status  = 'unprocessed'
        mcm_record.comment = "MCM run successful, results not saved to DB"
    session.flush()

    print "DRIVER: MCM run on code '{0}', McmRun ID {1} complete".format(mcm_record.prediction_code, mcm_record.id)