Python PrintLog类代码示例

    def partition(self, text):
        """private method - simulate zero width spaces for Japanese"""
        l = []
        r = ''
        last_n = ''
        for c in text:
            try:
                n = unicodedata.name(c).split()[0]
            except ValueError:
                n = 'NoName'
                PrintLog.message(u'No unicode name for: "{0:s}"'.format(c))

            if n in self.CJK:
                if '' != r:
                    l.append(r)
                r = c
                last_n = n

            elif last_n in self.CJK:
                if n in self.PUNCTUATION:
                    l.append(r + c)
                    r = ''
                    last_n = ''
                else:
                    l.append(r)
                    r = c
                    last_n = n
            else:
                r += c
                last_n = n

        if '' != r:
            l.append(r)
        return l

    def title(self, category, key, title, seek):
        if self.KEY_ARTICLE != key:
            if verbose:
                PrintLog.message('Non-article: {0:s}:{1:s}'.format(category,title))
            return False

        return True

 def resolve_redirects(self):
     """add redirect to article_index"""
     global verbose
     count = 0
     if verbose:
         PrintLog.message(u'Resolving redirects')
     else:
         pass
     for item in self.redirects:
         try:
             self.set_index(item, self.find(item)[:3] + (True,))
             count += 1
             if verbose and count % 1000 == 0:
                 PrintLog.message(u'Redirects resolved: {0:d}'.format(count))
             else:
                 pass
         except KeyError:
             PrintLog.message(u'Unresolved redirect: {0:s} -> {1:s}'.format(item, self.redirects[item]))
         except CycleError:
             PrintLog.message(u'Cyclic redirect: {0:s} -> {1:s}'.format(item, self.redirects[item]))
     if verbose:
         PrintLog.message(u'Total redirects resolved: {0:d}'.format(count))
     else:
         pass
     return count

def main():
    global verbose

    try:
        opts, args = getopt.getopt(sys.argv[1:],
                                   'hvd:e:',
                                   ['help',
                                    'verbose',
                                    'dir=',
                                    'extract=',
                                    ])
    except getopt.GetoptError as err:
        usage(err)

    verbose = False
    dir = 'image/enpedia'
    extract = None

    for opt, arg in opts:
        if opt in ('-v', '--verbose'):
            verbose = True
        elif opt in ('-h', '--help'):
            usage(None)
        elif opt in ('-d', '--dir'):
            dir = arg
        elif opt in ('-e', '--extract'):
            extract = arg
        else:
            usage('unhandled option: ' + opt)

    if not os.path.isdir(dir):
        usage('{0:s} is not a directory'.format(dir))

    idx_file = open(os.path.join(dir, "wiki.idx"), "rb")
    fnd_file = SegmentedFileReader(os.path.join(dir, "wiki{0:s}.fnd"))

    dat_format = os.path.join(dir, "wiki{0:d}.dat")

    index_min = 1
    index_max = struct.unpack('<I', idx_file.read(4))[0]

    PrintLog.message('Total index entries = {0:d}'.format(index_max))
    PrintLog.message('')

    for item in args:
        try:
            index_number = int(item.translate(None, ',_'), 0)
        except ValueError:
            usage('"{0:s}" is not numeric'.format(item))

        if index_number < index_min or index_number > index_max:
            usage('index: {0:d} is outdide [{1:d} .. {2:d}]'.format(index_number, index_min, index_max))

        process(index_number, idx_file, fnd_file, dat_format, extract)


    idx_file.close()
    fnd_file.close()

 def resolve_redirects(self):
     """add redirect to article_index"""
     count = 0
     for item in self.redirects:
         try:
             self.set_index(item, self.find(item)[:3] + (True,))
             count += 1
         except KeyError:
             PrintLog.message(u'Unresolved redirect: {0:s} -> {1:s}'.format(item, self.redirects[item]))
         except CycleError:
             PrintLog.message(u'Cyclic redirect: {0:s} -> {1:s}'.format(item, self.redirects[item]))
     return count

    def title(self, category, key, title, seek):
        global verbose
        global enable_templates

        if self.KEY_ARTICLE == key:
            return True

        if enable_templates and self.KEY_TEMPLATE == key:
            if verbose:
                PrintLog.message(u'Template Title: {0:s}'.format(unicode(title, 'utf-8')))
            return True

        return False

def process_article_text(id, count, title, text, newf):
    global verbose

    if verbose:
        PrintLog.message(u'[PA {0:d}] {1:s}'.format(count, title))

    text = TidyUp.article(text)

    if newf:
        newf.write('{0:d}:'.format(id))
        newf.write(title[1:].encode('utf-8'))  # We pad the title to force the database to import strings
        newf.write('\n__NOTOC__\n')
        newf.write(text.encode('utf-8') + '\n')
        newf.write('***EOF***\n')

    def body(self, category, key, title, text, seek):
        global verbose
        global error_flag

        title = self.convert(title).strip(u'\u200e\u200f')

        if self.KEY_TEMPLATE == key:
            if title not in self.ignored_templates:
                title = unicode(category, 'utf-8').capitalize() + ':' + upper_case_first_char(title)
                t_body = TidyUp.template(text)
                self.template_cursor.execute(u'insert or replace into templates (title, body) values(?, ?)',
                                             [u'~{0:d}~{1:s}'.format(self.file_id(), title), u'~' + t_body])
                self.template_count += 1
            return

        restricted = FilterWords.is_restricted(title) or FilterWords.is_restricted(text)

        self.article_count += 1

        # do closer inspection to see if really restricted
        if restricted:
            (restricted, bad_words) = FilterWords.find_restricted(text)

        if restricted:
            self.restricted_count += 1

        if self.article_count % 10000 == 0:
            start_time = time.time()
            PrintLog.message(u'Index: {0:7.2f}s {1:10d}'.format(start_time - self.time, self.article_count))
            self.time = start_time

        for t in self.language_processor.translate(title):
            generate_bigram(t)

        if verbose:
            if restricted:
                PrintLog.message(u'Restricted Title: {0:s}'.format(title))
                PrintLog.message(u'  --> {0:s}'.format(bad_words))
            else:
                PrintLog.message(u'Title: {0:s}'.format(title))
                pass

        character_count = len(text)
        self.total_character_count += character_count
        self.offsets[self.article_count] = (self.file_id(), title, seek, character_count, self.total_character_count)

        if self.set_index(title, (self.article_count, -1, restricted, False)): # -1 == place holder
            PrintLog.message(u'ERROR: Duplicate Title: {0:s}'.format(title))
            error_flag = True

def main():
    global verbose
    global INDEX_ITEM_SIZE
    global UINT32_SIZE

    try:
        opts, args = getopt.getopt(sys.argv[1:], 'hvo:f:p:', ['help', 'verbose', 'output=', 'prefix='])
    except getopt.GetoptError as err:
        usage(err)

    verbose = False
    in_format = 'pedia{0:d}.idx-tmp'
    out_name = 'pedia.idx'

    for opt, arg in opts:
        if opt in ('-v', '--verbose'):
            verbose = True
        elif opt in ('-h', '--help'):
            usage(None)
        elif opt in ('-p', '--prefix'):
            in_format = arg + '{0:d}.idx-tmp'
        elif opt in ('-o', '--output'):
            out_name = arg
        else:
            usage('unhandled option: ' + opt)

    out = open(out_name, 'wb')

    article_count = 0
    i = 0
    data = {}
    while True:
        in_name = in_format.format(i)
        if not os.path.isfile(in_name):
            break
        if verbose:
            PrintLog.message('combining: {0:s}'.format(in_name))
        data[i] = open(in_name, 'rb').read()
        article_count += len(data[i]) / INDEX_ITEM_SIZE
        i += 1

    out.write(struct.pack('<I', article_count))

    for j in range(i):
        out.write(data[j])

    out.close()

    PrintLog.message('Combined {0:d} files'.format(i))

def esc_code14(width, height, data):
    """output bitmap"""
    global g_starty, g_curr_face
    global output

    if 0 == width or 0 == height:
        return

    if len(data) != (width + 7) // 8 * height:
        PrintLog.message(u'Error: Corrupt Image')

    output.write(struct.pack('<BBH', 15, width, height) + data)

    lineh = get_lineheight(g_curr_face)

    if height >= lineh:
        g_starty += height - lineh + 3   # since Eric draws images 3px lower for alignment

def process(index_number, idx_file, fnd_file, dat_format):
    """dump the index and fnd file entries"""

    global verbose
    global sizes
    global distribution
    global dist_list
    global total
    global byte_count

    if verbose:
        PrintLog.message('Index number = {0:10n} 0x{0:08x}'.format(index_number))

    uint32_size = 4
    index_entry_size = 2 * uint32_size + 1

    index_offset = uint32_size + index_entry_size * (index_number - 1)

    idx_file.seek(index_offset)
    offset_dat, offset_fnd, file_id = struct.unpack('<2IB', idx_file.read(index_entry_size))

    data_file_name = dat_format.format(file_id)

    dat_file = open(data_file_name, 'rb')
    dat_file.seek(offset_dat)

    number_of_pages = struct.unpack('B', dat_file.read(1))[0]

    for i in range(0, number_of_pages):
        page_id, page_offset, page_length = struct.unpack('<3I', dat_file.read(12))
        restricted = 'Restricted' if (0 != page_offset & 0x80000000) else ''
        page_offset = page_offset & 0x7fffffff

        if page_id in sizes:
            PrintLog.message('DUP: {0:10n}'.format(page_id))

        sizes[page_id] = page_length
        for d in dist_list:
            if page_length <= d:
                distribution[d] += 1
                byte_count[d] += page_length
                total += 1
                break

    dat_file.close()

def output_pfx(filename):
    """output the pfx matrix"""
    global index_matrix

    PrintLog.message(u'Writing: {0:s}'.format(filename))
    start_time = time.time()
    out_f = open(filename, 'wb')
    list = '\0' + SearchKey.all_characters()
    for k1 in list:
        for k2 in list:
            for k3 in list:
                key = k1+k2+k3
                if key in index_matrix:
                    offset = index_matrix[key]
                else:
                    offset = 0
                out_f.write(struct.pack('<I', offset))

    out_f.close()
    PrintLog.message(u'Time: {0:7.1f}s'.format(time.time() - start_time))

def get_imgdata(imgfile, indent):
    try:
        img = gd.image(imgfile)
    except IOError as e:
        PrintLog.message(u'unable to open image file: {0:s} because: {1:s}'.format(imgfile, e))
        return (0, 0, r'')

    (width, height) = img.size()
    if width <= (LCD_WIDTH - LCD_IMG_MARGIN - indent):
        is_black = lambda x, y: (0, 0, 0) == img.colorComponents(img.getPixel((x, y)))
        h_range = range(0, width)
        v_range = range(0, height)
    elif height <= (LCD_WIDTH - LCD_IMG_MARGIN - indent):
        is_black = lambda x, y: (0, 0, 0) == img.colorComponents(img.getPixel((y, x)))
        v_range = range(0, width)
        h_range = range(height - 1, -1, -1)
        (width, height) = (height, width)
    else:
        PrintLog.message(u'image file: {0:s} is too big'.format(imgfile))
        return (0, 0, r'')

    data = ''
    for v in v_range:
        byte = 0
        bit_count = 8

        for h in h_range:
            if is_black(h, v):
                pixel = 1
            else:
                pixel = 0
            bit_count -= 1
            byte |= pixel << bit_count
            if 0 == bit_count:
                data += struct.pack('<B', byte)
                byte = 0
                bit_count = 8
        if 8 != bit_count:
            data += struct.pack('<B', byte)

    return (width, height, data)

    def redirect(self, category, key, title, rcategory, rkey, rtitle, seek):
        global whitespaces
        global verbose

        title = self.translate(title).strip(u'\u200e\u200f')

        rtitle = self.translate(rtitle).strip().strip(u'\u200e\u200f')
        rtitle = whitespaces.sub(' ', rtitle).strip().lstrip(':')

        if self.KEY_TEMPLATE == key:
            if title != rtitle:
                title = unicode(category, 'utf-8') + ':' + title.lower()
                rtitle = unicode(rcategory, 'utf-8') + ':' + rtitle.lower()
                self.template_cursor.execute(u'insert or replace into redirects (title, redirect) values(?, ?)',
                                             [u'~{0:d}~{1:s}'.format(self.file_id(), title),
                                              u'~{0:d}~{1:s}'.format(self.file_id(), rtitle)])

            self.template_redirect_count += 1
            return

        if self.KEY_ARTICLE != key or self.KEY_ARTICLE != rkey:
            if verbose:
                PrintLog.message(u'Non-article Redirect: {0:s}[{1:d}]:{2:s} ->  {3:s}[{4:d}]:{5:s}'
                                 .format(category, key, title, rcategory, rkey, rtitle))
            return

        if '' == rtitle:
            PrintLog.message(u'Empty Redirect for: {0:s}[{1:d}]:{2:s}'.format(category, key, title))
        else:
            self.redirects[title] = rtitle
            self.redirect_count += 1
            if verbose:
                PrintLog.message(u'Redirect: {0:s}[{1:d}]:{2:s} ->  {3:s}[{4:d}]:{5:s}'
                                 .format(category, key, title, rcategory, rkey, rtitle))

    def handle_charref(self, name):
        """handle &#DDDD; &#xXXXX;"""

        if 0 == len(name):
            return

        if 'x' == name[0] or 'X' == name[0]:
            try:
                value = int(name[1:], 16)
            except ValueError:
                PrintLog.message(u'charref: "{0:s}" is not hexadecimal'.format(name))
                return

        elif name.isdigit():
            try:
                value = int(name)
            except ValueError:
                PrintLog.message(u'charref: "{0:s}" is not decimal'.format(name))
                return

        try:
            c = unichr(value)
        except ValueError:
            PrintLog.message(u'charref: "{0:d}" is not convertible to unicode'.format(value))
            c = '?'
        self.handle_data(c)