Python csv.Sniffer方法代码示例

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def _modify_column_headers(tmp_file_location,
                               custom_dimension_header_mapping):
        logger.info('Modifying column headers to be compatible for data upload')
        with open(tmp_file_location, 'r') as check_header_file:
            has_header = csv.Sniffer().has_header(check_header_file.read(1024))
            if has_header:
                with open(tmp_file_location, 'r') as read_file:
                    reader = csv.reader(read_file)
                    headers = next(reader)
                    new_headers = []
                    for header in headers:
                        if header in custom_dimension_header_mapping:
                            header = custom_dimension_header_mapping.get(header)
                        new_header = 'ga:' + header
                        new_headers.append(new_header)
                    all_data = read_file.readlines()
                    final_headers = ','.join(new_headers) + '\n'
                    all_data.insert(0, final_headers)
                    with open(tmp_file_location, 'w') as write_file:
                        write_file.writelines(all_data)
            else:
                raise NameError('CSV does not contain headers, please add them '
                                'to use the modify column headers functionality') 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def csv_to_dmatrix(string_like, dtype=None):  # type: (str) -> xgb.DMatrix
    """Convert a CSV object to a DMatrix object.
    Args:
        string_like (str): CSV string. Assumes the string has been stripped of leading or trailing newline chars.
        dtype (dtype, optional):  Data type of the resulting array. If None, the dtypes will be determined by the
                                        contents of each column, individually. This argument can only be used to
                                        'upcast' the array.  For downcasting, use the .astype(t) method.
    Returns:
        (xgb.DMatrix): XGBoost DataMatrix
    """
    sniff_delimiter = csv.Sniffer().sniff(string_like.split('\n')[0][:512]).delimiter
    delimiter = ',' if sniff_delimiter.isalnum() else sniff_delimiter
    logging.info("Determined delimiter of CSV input is \'{}\'".format(delimiter))

    np_payload = np.array(list(map(lambda x: _clean_csv_string(x, delimiter), string_like.split('\n')))).astype(dtype)
    return xgb.DMatrix(np_payload) 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def from_csv(fp, field_names = None, **kwargs):

    dialect = csv.Sniffer().sniff(fp.read(1024))
    fp.seek(0)
    reader = csv.reader(fp, dialect)

    table = PrettyTable(**kwargs)
    if field_names:
        table.field_names = field_names
    else:
        if py3k:
            table.field_names = [x.strip() for x in next(reader)]
        else:
            table.field_names = [x.strip() for x in reader.next()]

    for row in reader:
        table.add_row([x.strip() for x in row])

    return table 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def test_delimiters(self):
        sniffer = csv.Sniffer()
        dialect = sniffer.sniff(self.sample3)
        # given that all three lines in sample3 are equal,
        # I think that any character could have been 'guessed' as the
        # delimiter, depending on dictionary order
        self.assertIn(dialect.delimiter, self.sample3)
        dialect = sniffer.sniff(self.sample3, delimiters="?,")
        self.assertEqual(dialect.delimiter, "?")
        dialect = sniffer.sniff(self.sample3, delimiters="/,")
        self.assertEqual(dialect.delimiter, "/")
        dialect = sniffer.sniff(self.sample4)
        self.assertEqual(dialect.delimiter, ";")
        dialect = sniffer.sniff(self.sample5)
        self.assertEqual(dialect.delimiter, "\t")
        dialect = sniffer.sniff(self.sample6)
        self.assertEqual(dialect.delimiter, "|")
        dialect = sniffer.sniff(self.sample7)
        self.assertEqual(dialect.delimiter, "|")
        self.assertEqual(dialect.quotechar, "'")
        dialect = sniffer.sniff(self.sample8)
        self.assertEqual(dialect.delimiter, '+')
        dialect = sniffer.sniff(self.sample9)
        self.assertEqual(dialect.delimiter, '+')
        self.assertEqual(dialect.quotechar, "'") 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def __init__(self, fname, labels):
        """
        Initialize the corpus from a file.
        `labels` = are class labels present in the input file? => skip the first column

        """
        logger.info("loading corpus from %s" % fname)
        self.fname = fname
        self.length = None
        self.labels = labels

        # load the first few lines, to guess the CSV dialect
        head = ''.join(itertools.islice(open(self.fname), 5))
        self.headers = csv.Sniffer().has_header(head)
        self.dialect = csv.Sniffer().sniff(head)
        logger.info("sniffed CSV delimiter=%r, headers=%s" % (self.dialect.delimiter, self.headers)) 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def test_delimiters(self):
        sniffer = csv.Sniffer()
        dialect = sniffer.sniff(self.sample3)
        # given that all three lines in sample3 are equal,
        # I think that any character could have been 'guessed' as the
        # delimiter, depending on dictionary order
        self.assertIn(dialect.delimiter, self.sample3)
        dialect = sniffer.sniff(self.sample3, delimiters="?,")
        self.assertEqual(dialect.delimiter, "?")
        dialect = sniffer.sniff(self.sample3, delimiters="/,")
        self.assertEqual(dialect.delimiter, "/")
        dialect = sniffer.sniff(self.sample4)
        self.assertEqual(dialect.delimiter, ";")
        dialect = sniffer.sniff(self.sample5)
        self.assertEqual(dialect.delimiter, "\t")
        dialect = sniffer.sniff(self.sample6)
        self.assertEqual(dialect.delimiter, "|")
        dialect = sniffer.sniff(self.sample7)
        self.assertEqual(dialect.delimiter, "|")
        self.assertEqual(dialect.quotechar, "'") 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def __init__(
            self,
            csv_file_path,
            converter=None,
            callbacks={}
    ):
        super().__init__(converter=converter, callbacks=callbacks)

        self.csv_file_path = csv_file_path
        self.csvfile = open(self.csv_file_path)
        self.sniffer = csv.Sniffer()
        self.dialect = self.sniffer.sniff(self.csvfile.read(1024))
        self.csvfile.seek(0)
        self.has_header = self.sniffer.has_header(self.csvfile.read(1024))
        self.csvfile.seek(0)
        self.reader = csv.DictReader(self.csvfile, dialect=self.dialect)
        if self.has_header:
            print('=======')
            print(self.reader.fieldnames)
            print('=======')
        else:
            # should raise error?
            pass 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def __init__(self, path, sep=' ', **kwargs):
        self._node_ids = None
        self._min_time = None
        self._max_time = None
        # self._dt = None

        try:
            # check to see if file contains headers
            with open(path, 'r') as csvfile:
                sniffer = csv.Sniffer()
                has_headers = sniffer.has_header(csvfile.read(1024))
        except Exception:
            has_headers = True

        self._spikes_df = pd.read_csv(path, sep=sep, header=0 if has_headers else None)

        if not has_headers:
            self._spikes_df.columns = csv_headers[:self._spikes_df.shape[1]]

        if col_population not in self._spikes_df.columns:
            pop_name = kwargs.get(col_population, pop_na)
            self._spikes_df[col_population] = pop_name 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def __init__(self, path, sep=' ', **kwargs):
        self._n_spikes = None
        self._populations = None

        try:
            # check to see if file contains headers
            with open(path, 'r') as csvfile:
                sniffer = csv.Sniffer()
                has_headers = sniffer.has_header(csvfile.read(1024))
        except Exception:
            has_headers = True

        self._spikes_df = pd.read_csv(path, sep=sep, header=0 if has_headers else None)

        if not has_headers:
            self._spikes_df.columns = csv_headers[0::2]

        if col_population not in self._spikes_df.columns:
            pop_name = kwargs.get(col_population, pop_na)
            self._spikes_df[col_population] = pop_name

        # TODO: Check all the necessary columns exits
        self._spikes_df = self._spikes_df[csv_headers] 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def get_dialect(file_obj, sample_size=1024*1024):
    try:
        sample = file_obj.read(sample_size)
    except StopIteration:
        sample = file_obj.readline()
    file_obj.seek(0)

    sniffer = csv.Sniffer()
    dialect = sniffer.sniff(sample)

    result = {
        'delimiter': dialect.delimiter,
        'doublequote': dialect.doublequote,
        'escapechar': dialect.escapechar,
        'lineterminator': dialect.lineterminator,
        'quotechar': dialect.quotechar,
    }
    return result 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def csvfile_to_wb(csv_filename):
    '''Open a CSV file and return an openpyxl workbook.'''

    logger.log(
        DEBUG_DETAILED,
        'Converting CSV file {} into an XLSX workbook.'.format(csv_filename))

    with open(csv_filename) as csv_file:
        dialect = csv.Sniffer().sniff(csv_file.read())
        if USING_PYTHON2:
            for attr in dir(dialect):
                a = getattr(dialect, attr)
                if type(a) == unicode:
                    setattr(dialect, attr, bytes(a))
        csv_file.seek(0)
        reader = csv.reader(csv_file, dialect)
        wb = pyxl.Workbook()
        ws = wb.active
        for row_index, row in enumerate(reader, 1):
            for column_index, cell in enumerate(row, 1):
                if cell not in ('', None):
                    ws.cell(row=row_index, column=column_index).value = cell
    return (wb, dialect) 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def test_delimited(filepath):
    import csv
    if six.PY3:
        handle = open(filepath, 'r', newline='')
    else:
        handle = open(filepath, 'rb')
    with handle as csv_file:
        try:
            dialect = csv.Sniffer().sniff(csv_file.read(1024*16), delimiters=',\t')
        except Exception as e:
            return False, None
        csv_file.seek(0)
        reader = csv.reader(csv_file, dialect)
        rows = []
        try:
            for index, entry in enumerate(reader):
                if index == 5:
                    break
                rows.append(entry)
        except Exception as e:
            return False, None
        return True, rows 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def get_column_value_list(filenames, column_name, debug=False):
        """Get the values of the given column from all filenames.

        Function that given a set of filenames returns the list of values
        concatenating all the columns with name "column_name"
        :param filenames: List of filenames
        :param column_name: Column name to search.
        :param debug: Boolean controlling the log messages.
        :return: List of values.
        """
        to_return = []
        for file_name in filenames:

            # Open the file for reading
            file_in = codecs.open(file_name, 'rU')
            dialect = csv.Sniffer().sniff(file_in.read(1024))
            file_in.seek(0)
            data_in = csv.reader(file_in, dialect=dialect, delimiter=str(','))

            if debug:
                print('Parsing file ' + file_name)

            to_return += process_csv_file(data_in, column_name)

        return to_return 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def sniff_for_header(file_path, sep="\t", comment="#"):
    """
    Warning: reads the entire file into a StringIO buffer!

    """
    with open(file_path, "r") as f:
        buf = io.StringIO(f.read())

    sample_lines = []
    for line in buf:
        if not line.startswith(comment):
            sample_lines.append(line)
            break
    for _ in range(10):
        sample_lines.append(buf.readline())
    buf.seek(0)

    has_header = csv.Sniffer().has_header("\n".join(sample_lines))
    if has_header:
        names = sample_lines[0].strip().split(sep)
    else:
        names = None

    return buf, names 

# 需要导入模块: import csv [as 别名]
# 或者: from csv import Sniffer [as 别名]
def load(self, filename: str) -> None:
        keys = None
        types = None

        with open(filename, newline='') as csvfile:
            dialect = csv.Sniffer().sniff(csvfile.read())
            csvfile.seek(0)
            reader = csv.reader(csvfile, dialect)
            for row in reader:
                if len(row) == 0:
                    continue
                if keys is None:
                    if len(row[0].split(":")) == 2:
                        keys = [r.split(':')[0] for r in row]
                        types = [r.split(':')[1] for r in row]  # NOTE: I don't use type anymore
                    else:
                        keys = row
                        types = ["None" for r in row]
                else:
                    self.items.append(Metadata(keys, row, types))