Python Store.save方法代码示例

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import save [as 别名]
def go_store():
    store = Store()

    store.remove(None)
    store.add(323, 2, 10, 88, 78, 415, 89, 189)
    store.remove(78)

    store.save()
    store.load()

    print store.grep('\d*[02468]$')
    print store.find(89, -1, 415)

    for item in store.items():
        print item

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import save [as 别名]
 def add(cls, data):
     '''
     :param data: str
     :returns: boolean
     '''
     cls.check_for_heartbeat_request(data)
     parsed = Parse(data)
     if not parsed:
         logging.warning('Unable to parse log message: {}'.format(data))
         return False
     filters = Filters.get()
     matched_filters = Filters_checker.check(
         filters=filters, 
         parsed=parsed,
     )
     store = Store(parsed=parsed, matched_filters=matched_filters)
     if not store.save():
         return False
     return True

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import save [as 别名]

#.........这里部分代码省略.........
        self.parser.add_option('-u', '--url', nargs=1, help="process just the given URL")
        self.parser.add_option('-i', '--ini-file', default="churnalism.cfg", nargs=1, help="filename for connection settings [default: %default]")


    def main(self):
        """ set everything up, then invoke go() """

        (options, args) = self.parser.parse_args()

        log_level = logging.ERROR
        if options.debug:
            log_level = logging.DEBUG
        elif options.verbose:
            log_level = logging.INFO
        logging.basicConfig(level=log_level)    #, format='%(message)s')


        if options.test:
            self.store = DummyStore(self.name, self.doc_type)
        else:
            # load in config file for real run
            config = ConfigParser.ConfigParser()
            config.readfp(open(options.ini_file))
            auth_user = config.get("DEFAULT",'user')
            auth_pass = config.get("DEFAULT",'pass')
            server = config.get("DEFAULT",'server')

            self.store = Store(self.name, self.doc_type, auth_user=auth_user, auth_pass=auth_pass, server=server)


        if options.cache:
            logging.info("using .cache")
            opener = urllib2.build_opener(CacheHandler(".cache"))
            urllib2.install_opener(opener)

        self.go(options)


    def go(self,options):
        """ perform the actual scraping

        default implementation is to just call find_latest and process the discovered press releases.
        But it's likely derived classes will want to handle custom options for fetching historical
        data

        see prnewswire for an example.
        """


        if options.url:
            urls = [options.url,]
        else:
            urls = self.find_latest()

        self.process_batch(urls)


    def process_batch(self, urls):
        """ run through a list of urls, fetching, extracting and storing each in turn """

        # cull out ones we've got
        n_before = len(urls)
        urls = [url for url in urls if not self.store.already_got(url)]
        logging.info("processing %d urls (%d are new)", n_before, len(urls))

        err_cnt = 0
        try:

            for url in urls:
                try:
                    logging.debug("fetch %s",url)
                    response = urllib2.urlopen(url)
                    html = response.read()
                    # TODO: maybe just skip ones which redirect to other domains?
                    if response.geturl() != url:
                        logging.warning("Redirect detected %s => %s",url,response.geturl())
                    press_release = self.extract(html, url)

                    # encode text fields
                    # TODO: use isinstance(...,unicode) instead
                    for f in ('url','title','source','text','location','language','topics'):
                        if f in press_release:
                            press_release[f] = press_release[f].encode('utf-8')
                    self.store.add(press_release)

                except Exception as e:
                    logging.error("failed on %s: %s %s",url,e.__class__,e)
                    err_cnt += 1
        finally:
            self.store.save()


    def find_latest(self):
        """ obtain the list of "latest" press releases, whatever that means for a given target """
        return []


    def extract(self,html,url):
        """ extract a single downloaded press release """
        assert False    # need to implement in derived class!

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import save [as 别名]
class GUI(object):
    def __init__(self, filename):
        self.filename = filename
        self.store = Store()
        self.root = tkinter.Tk()
        self.passphrase = None
        self.selected = None
        self.value_changing = False

        left_frame = tkinter.Frame(self.root)
        left_frame.pack(side=tkinter.LEFT, fill=tkinter.Y)
        left_frame.rowconfigure(0, weight=1)

        scroll = tkinter.Scrollbar(left_frame)
        self.keys = tkinter.Listbox(left_frame, exportselection=0)

        scroll.grid(row=0, column=1, sticky=tkinter.N+tkinter.S)
        self.keys.grid(row=0, column=0, sticky=tkinter.N+tkinter.S)

        scroll.config(command=self.keys.yview)
        self.keys.config(yscrollcommand=scroll.set)

        self.keys.bind("<ButtonRelease>", self.on_key_selected)
        self.keys.bind("<KeyRelease>", self.on_key_selected)

        add = tkinter.Button(
            left_frame,
            text="Add",
            command=self.on_add_entry,
        )
        add.grid(row=1, columnspan=2, sticky=tkinter.E+tkinter.W)

        self.value_box = tkinter.Text(self.root)
        self.value_box.pack(side=tkinter.LEFT, fill=tkinter.BOTH)
        self.value_box.bind('<<Modified>>', self.on_value_changed)

        self.root.bind('<Control-s>', self.on_save)

    def run(self):
        if os.path.exists(self.filename):
            dialog = PassphraseDialog(self.root, title='Passphrase')
            if not dialog.run():
                return
            self.passphrase = dialog.passphrase
            self.store.load(self.passphrase, self.filename)
            self.show_keys()
        else:
            dialog = PassphraseDialog(self.root, title='Passphrase', confirm=True)
            if not dialog.run():
                return
            self.passphrase = dialog.passphrase
            self.store.create(self.passphrase)

        self.update_modified()
        self.root.mainloop()

    def on_key_selected(self, *_args):
        selected = self.keys.curselection()
        if not selected:
            self.selected = None
            return
        idx = int(selected[0])
        self.selected = self.keys.get(idx)
        value = self.store.entries[self.selected]
        self.value_changing = True
        self.value_box.delete(1.0, tkinter.END)
        self.value_box.insert(1.0, value)
        self.value_changing = False

    def on_add_entry(self, *_args):
        dialog = Dialog(self.root, title='Add', label='name')
        if not dialog.run():
            return
        key = dialog.value
        self.store.entries[key] = ''
        self.store.modified = True
        self.update_modified()
        keys = self.show_keys()
        self.keys.select_clear(0, tkinter.END)
        self.keys.select_set(keys.index(key))
        self.on_key_selected()

    def on_value_changed(self, *_args):
        # guard against getting called during key selection changes
        # and also when the modified flag is reset to False.
        if self.value_changing:
            self.root.call(self.value_box, 'edit', 'modified', 0)
            return
        self.value_changing = True
        self.root.call(self.value_box, 'edit', 'modified', 0)
        self.value_changing = False

        if self.selected:
            value = self.value_box.get(1.0, tkinter.END)[:-1]
            if self.store.entries[self.selected] != value:
                self.store.entries[self.selected] = value
                self.store.modified = True
                self.update_modified()

    def on_save(self, *_args):
#.........这里部分代码省略.........

# 需要导入模块: from store import Store [as 别名]
# 或者: from store.Store import save [as 别名]
class CLI(object):
    def __init__(self, filename):
        self.filename = filename
        self.store = Store()

        if os.path.exists(filename):
            self.passphrase = self.get_passphrase()
            self.store.load(self.passphrase, filename)
        else:
            self.passphrase = self.get_passphrase(confirm=True)
            self.store.create(self.passphrase)

        self.commands = {
            'ls' : (self.cmd_list, 'l'),
            'cat' : (self.cmd_cat, 'c:'),
            'set' : (self.cmd_set, 's:'),
            'rm' : (self.cmd_remove, 'r:'),
            'save' : (self.cmd_save, ' '),
            'change_passphrase': (self.cmd_change_passphrase, ' '),
        }


    def cmd_list(self):
        print '\n'.join(sorted(self.store.entries.iterkeys()))

    def cmd_cat(self, label):
        entry = self.store.entries.get(label)
        if entry is not None:
            print entry
        else:
            print 'ERROR: label not found'

    def cmd_set(self, label):
        value = ''
        while True:
            line = sys.stdin.readline()
            if line[0] == '\r' or line[0] == '\n':
                break
            value += line
        self.store.entries[label] = value.strip()
        self.store.modified = True

    def cmd_remove(self, label):
        try:
            del self.store.entries[label]
            self.store.modified = True
        except KeyError:
            print 'ERROR: label not found'

    def cmd_save(self):
        self.store.save(self.filename)

    def cmd_change_passphrase(self):
        current = self.get_passphrase(prompt="Current passphrase: ")
        if current != self.passphrase:
            print 'Incorrect, passphrase not changed.'
            return
        new_passphrase = self.get_passphrase(confirm=True, allow_blank=True)
        if not new_passphrase:
            print 'Passphrase not changed.'
        else:
            self.passphrase = new_passphrase
            self.store.change_passphrase(new_passphrase)

    def completer(self, text, state):
        parts = readline.get_line_buffer().split(' ')
        idx = len(parts)

        if idx == 1:
            matches = 0
            for command, attrs in self.commands.items():
                if command.startswith(text):
                    matches += 1
                    if matches > state:
                        if attrs[-1][-1] == ':':
                            command += ' '
                        return command
        elif idx == 2:
            matches = 0
            for label in self.store.entries.iterkeys():
                if label.startswith(text):
                    matches += 1
                    if matches > state:
                        return label

    def get_passphrase(self, prompt=None, confirm=False, allow_blank=False):
        while True:
            passphrase = getpass(prompt or "Passphrase: ")
            if not passphrase:
                if allow_blank:
                    return
                continue

            if confirm:
                again = getpass("Confirm passphrase: ")
                if passphrase != again:
                    print 'Passphrases did not match, please try again.'
                    continue

            break
#.........这里部分代码省略.........